def splitJobList(jobList,
src,
targetSet,
hidden=False,
callback=ChangesetCallback()):
log.debug("Changeset Key conflict detected; splitting job further...")
jobs = {}
for job in jobList:
name = job[0]
if ':' in name:
name = name.split(':')[0]
l = jobs.setdefault(name, [])
l.append(job)
i = 0
for smallJobList in jobs.itervalues():
(outFd, tmpName) = util.mkstemp()
os.close(outFd)
log.debug(
"jobsplit %d of %d %s" %
(i + 1, len(jobs), displayBundle([(0, x) for x in smallJobList])))
src.createChangeSetFile(smallJobList,
tmpName,
recurse=False,
callback=callback,
mirrorMode=True)
for target in targetSet:
target.commitChangeSetFile(tmpName,
hidden=hidden,
callback=callback)
os.unlink(tmpName)
callback.done()
i += 1
return
def buildBundles(sourceRepos, target, troveList, absolute=False):
bundles = []
log.debug("grouping %d troves based on version and flavor", len(troveList))
groupList = groupTroves(troveList)
log.debug("building grouped job list")
bundles = buildJobList(sourceRepos, target.repo, groupList, absolute)
return bundles
def recurseTrove(sourceRepos,
name,
version,
flavor,
callback=ChangesetCallback()):
global recursedGroups
assert (trove.troveIsGroup(name))
# there's nothing much we can recurse from the source
if name.endswith(":source"):
return []
# avoid grabbing the same group multiple times
if (name, version, flavor) in recursedGroups:
return []
log.debug("recursing group trove: %s=%s[%s]" % (name, version, flavor))
groupCs = sourceRepos.createChangeSet([(name, (None, None),
(version, flavor), True)],
withFiles=False,
withFileContents=False,
recurse=False,
callback=callback)
recursedGroups.add((name, version, flavor))
ret = []
for troveCs in groupCs.iterNewTroveList():
for name, ops in troveCs.iterChangedTroves(True, True):
for oper, version, flavor, byDefault in ops:
if oper != '-':
ret.append((name, version, flavor))
return ret
def buildBundles(sourceRepos, target, troveList, absolute=False):
bundles = []
log.debug("grouping %d troves based on version and flavor", len(troveList))
groupList = groupTroves(troveList)
log.debug("building grouped job list")
bundles = buildJobList(sourceRepos, target.repo, groupList, absolute)
return bundles
def _createDirs(self):
for dir, mode, uid, gid in self.dirsToAdd:
dir = self.root + dir
log.debug("creating chroot:%s", dir)
util.mkdirChain(dir)
if mode:
os.chmod(dir, mode)
if (uid or gid) and not os.getuid():
os.chown(dir, uid, gid)
def _createDirs(self):
for dir, mode, uid, gid in self.dirsToAdd:
dir = self.root + dir
log.debug("creating chroot:%s", dir)
util.mkdirChain(dir)
if mode:
os.chmod(dir, mode)
if (uid or gid) and not os.getuid():
os.chown(dir, uid, gid)
def addTroveList(self, tl):
# Filter out troves which are already in the local repository. Since
# the marks aren't distinct (they increase, but not monotonically), it's
# possible that something new got committed with the same mark we
# last updated to, so we have to look again at all of the troves in the
# source repository with the last mark which made it into our target.
present = self.repo.hasTroves([x[1] for x in tl], hidden=True)
ret = [x for x in tl if not present[x[1]]]
log.debug("%s found %d troves not present", self.name, len(ret))
return ret
def addTroveList(self, tl):
# Filter out troves which are already in the local repository. Since
# the marks aren't distinct (they increase, but not monotonically), it's
# possible that something new got committed with the same mark we
# last updated to, so we have to look again at all of the troves in the
# source repository with the last mark which made it into our target.
present = self.repo.hasTroves([ x[1] for x in tl ], hidden = True)
ret = [ x for x in tl if not present[x[1]] ]
log.debug("%s found %d troves not present", self.name, len(ret))
return ret
def mirrorGPG(self, src, host):
if self.__gpg.has_key(host):
return
keyList = src.getNewPGPKeys(host, -1)
self.__gpg[host] = keyList
if not len(keyList):
return
log.debug("%s adding %d gpg keys", self.name, len(keyList))
if self.test:
return
self.repo.addPGPKeyList(self.cfg.host, keyList)
def mirrorGPG(self, src, host):
if self.__gpg.has_key(host):
return
keyList = src.getNewPGPKeys(host, -1)
self.__gpg[host] = keyList
if not len(keyList):
return
log.debug("%s adding %d gpg keys", self.name, len(keyList))
if self.test:
return
self.repo.addPGPKeyList(self.cfg.host, keyList)
def _compare(src, dst):
srcName, srcSet = src
dstName, dstSet = dst
counter = 0
for x in srcSet.difference(dstSet):
log.debug(" - %s %s " % (srcName, x))
counter += 1
for x in dstSet.difference(srcSet):
log.debug(" + %s %s" % (dstName, x))
counter += 1
return counter
def _compare(src, dst):
srcName, srcSet = src
dstName, dstSet = dst
counter = 0
for x in srcSet.difference(dstSet):
log.debug(" - %s %s " % (srcName, x))
counter += 1
for x in dstSet.difference(srcSet):
log.debug(" + %s %s" % (dstName, x))
counter += 1
return counter
def mainWorkflow(cfg = None, callback=ChangesetCallback(),
test=False, sync=False, infoSync=False,
checkSync=False, fastSync=False):
import fcntl
if cfg.lockFile:
try:
log.debug('checking for lock file')
lock = open(cfg.lockFile, 'w')
fcntl.lockf(lock, fcntl.LOCK_EX|fcntl.LOCK_NB)
except IOError:
log.warning('lock held by another process, exiting')
return
# need to make sure we have a 'source' section
if not cfg.hasSection('source'):
log.debug("ERROR: mirror configuration file is missing a [source] section")
raise RuntimeError("Mirror configuration file is missing a [source] section")
sourceRepos = _getMirrorClient(cfg, 'source')
# Optional reference repository
if cfg.hasSection('reference'):
refRepos = _getMirrorClient(cfg, 'reference')
else:
refRepos = sourceRepos
# we need to build a target repo client for each of the "target*"
# sections in the config file
targets = []
for name in cfg.iterSectionNames():
if not name.startswith("target"):
continue
target = _getMirrorClient(cfg, name)
target = TargetRepository(target, cfg, name, test=test)
targets.append(target)
# checkSync is a special operation...
if checkSync:
return checkSyncRepos(cfg, refRepos, targets)
# we pass in the sync flag only the first time around, because after
# that we need the targetRepos mark to advance accordingly after being
# reset to -1
callAgain = mirrorRepository(sourceRepos, targets, cfg,
test = test, sync = sync,
syncSigs = infoSync,
callback = callback,
fastSync = fastSync,
referenceRepos=refRepos,
)
while callAgain:
callAgain = mirrorRepository(sourceRepos, targets, cfg,
test = test, callback = callback,
fastSync = fastSync,
referenceRepos=refRepos,
)
def addUser(self, name, uid, gid=None, home=None, shell='/bin/bash'):
"""
Adds user that must be in /etc/passwd when root is instantiated
"""
if gid is None:
gid = uid
if gid == uid:
self.groupsToSupport.append((name, gid, []))
if home is None:
home = '/home/%s' % name
log.debug("adding user %s (%s,%s) home=%s", name, uid, gid, home)
self.usersToSupport.append((name, uid, gid, home, shell))
def _copyFiles(self):
for (sourceFile, targetFile, mode) in self.filesToCopy:
log.debug("copying file %s into chroot:%s", sourceFile, targetFile)
try:
target = self.root + targetFile
target = os.path.realpath(target)
util.mkdirChain(os.path.dirname(target))
shutil.copy(sourceFile, target)
if mode is not None:
os.chmod(target, mode)
except (IOError, OSError), e:
raise errors.OpenError("Could not copy in file %s to %s: %s" % (sourceFile, targetFile, e))
def addUser(self, name, uid, gid=None, home=None, shell="/bin/bash"):
"""
Adds user that must be in /etc/passwd when root is instantiated
"""
if gid is None:
gid = uid
if gid == uid:
self.groupsToSupport.append((name, gid, []))
if home is None:
home = "/home/%s" % name
log.debug("adding user %s (%s,%s) home=%s", name, uid, gid, home)
self.usersToSupport.append((name, uid, gid, home, shell))
def _copyFiles(self):
for (sourceFile, targetFile, mode) in self.filesToCopy:
log.debug("copying file %s into chroot:%s", sourceFile, targetFile)
try:
target = self.root + targetFile
target = os.path.realpath(target)
util.mkdirChain(os.path.dirname(target))
shutil.copy(sourceFile, target)
if mode is not None:
os.chmod(target, mode)
except (IOError, OSError), e:
raise errors.OpenError('Could not copy in file %s to %s: %s' %
(sourceFile, targetFile, e))
def commitChangeSetFile(self, filename, hidden, callback):
if self.test:
return 0
callback.setPrefix(self.name + ": ")
t1 = time.time()
ret = self.repo.commitChangeSetFile(filename, mirror=True, hidden=hidden,
callback=callback)
t2 = time.time()
callback.done()
hstr = ""
if hidden: hstr = "hidden "
log.debug("%s %scommit (%.2f sec)", self.name, hstr, t2-t1)
return ret
def mirrorRemoved(sourceRepos, targetRepos, troveSet, test = False, callback = ChangesetCallback()):
if not troveSet:
return 0
log.debug("checking on %d removed troves", len(troveSet))
# these removed troves better exist on the target
present = targetRepos.hasTroves(list(troveSet))
missing = [ x for x in troveSet if not present[x] ]
# we can not have any "missing" troves while we mirror removals
for t in missing:
log.warning("Mirroring removed trove: valid trove not found on target: %s", t)
troveSet.remove(t)
# for the remaining removed troves, are any of them already mirrored?
jobList = [ (name, (None, None), (version, flavor), True) for
(name, version, flavor) in troveSet ]
cs = targetRepos.createChangeSet(jobList, recurse=False, withFiles=False,
withFileContents=False, callback=callback)
for trvCs in cs.iterNewTroveList():
if trvCs.getType() == trove.TROVE_TYPE_REMOVED:
troveSet.remove(trvCs.getNewNameVersionFlavor())
log.debug("mirroring %d removed troves", len(troveSet))
if not troveSet:
return 0
jobList = [ (name, (None, None), (version, flavor), True) for
(name, version, flavor) in troveSet ]
log.debug("mirroring removed troves %s" % (displayJobList(jobList),))
# grab the removed troves changeset
cs = sourceRepos.createChangeSet(jobList, recurse = False,
withFiles = False, withFileContents = False,
callback = callback)
log.debug("committing")
targetRepos.commitChangeSet(cs, mirror = True, callback = callback)
callback.done()
return len(jobList)
def mirrorRemoved(sourceRepos, targetRepos, troveSet, test = False, callback = ChangesetCallback()):
if not troveSet:
return 0
log.debug("checking on %d removed troves", len(troveSet))
# these removed troves better exist on the target
present = targetRepos.hasTroves(list(troveSet))
missing = [ x for x in troveSet if not present[x] ]
# we can not have any "missing" troves while we mirror removals
for t in missing:
log.warning("Mirroring removed trove: valid trove not found on target: %s", t)
troveSet.remove(t)
# for the remaining removed troves, are any of them already mirrored?
jobList = [ (name, (None, None), (version, flavor), True) for
(name, version, flavor) in troveSet ]
cs = targetRepos.createChangeSet(jobList, recurse=False, withFiles=False,
withFileContents=False, callback=callback)
for trvCs in cs.iterNewTroveList():
if trvCs.getType() == trove.TROVE_TYPE_REMOVED:
troveSet.remove(trvCs.getNewNameVersionFlavor())
log.debug("mirroring %d removed troves", len(troveSet))
if not troveSet:
return 0
jobList = [ (name, (None, None), (version, flavor), True) for
(name, version, flavor) in troveSet ]
log.debug("mirroring removed troves %s" % (displayJobList(jobList),))
# grab the removed troves changeset
cs = sourceRepos.createChangeSet(jobList, recurse = False,
withFiles = False, withFileContents = False,
callback = callback)
log.debug("committing")
targetRepos.commitChangeSet(cs, mirror = True, callback = callback)
callback.done()
return len(jobList)
def commitChangeSetFile(self, filename, hidden, callback):
if self.test:
return 0
callback = copy.copy(callback)
callback.setPrefix(self.name + ": ")
t1 = time.time()
ret = self.repo.commitChangeSetFile(filename, mirror=True, hidden=hidden,
callback=callback)
t2 = time.time()
callback.done()
hstr = ""
if hidden: hstr = "hidden "
log.debug("%s %scommit (%.2f sec)", self.name, hstr, t2-t1)
return ret
def _copyDirs(self):
for (sourceDir, targetDir) in self.dirsToCopy:
targetDir = self.root + targetDir
if os.path.exists(targetDir):
if os.path.islink(targetDir):
os.unlink(targetDir)
else:
util.rmtree(targetDir)
util.mkdirChain(os.path.realpath(os.path.dirname(targetDir)))
log.debug("copying dir %s into chroot:%s", sourceDir, targetDir)
try:
shutil.copytree(sourceDir, os.path.realpath(targetDir))
except shutil.Error, e:
errorList = "\n".join("cannot copy %s to %s: %s" % x for x in e.args[0])
raise errors.OpenError("Could not copy in directory %s:\n%s" % (sourceDir, errorList))
def __init__(self, client):
log.debug("loading system model cache");
troveCache = trovecache.TroveCache(None)
troveCache.load(client.cfg.dbPath + '/modelcache')
model = cml.CML(client.cfg)
troveSet = client.cmlGraph(model)
troveSet.g.realize(modelupdate.CMLActionData(troveCache,
client.cfg.flavor[0],
client.getRepos(), client.cfg))
self.troveTups = set()
for withFiles, trv in troveCache.cache.values():
for nvf in trv.iterTroveList(strongRefs = True, weakRefs = True):
self.troveTups.add(nvf)
def __init__(self, client):
log.debug("loading system model cache")
troveCache = trovecache.TroveCache(None)
troveCache.load(client.cfg.dbPath + '/modelcache')
model = cml.CML(client.cfg)
troveSet = client.cmlGraph(model)
troveSet.g.realize(
modelupdate.CMLActionData(troveCache, client.cfg.flavor[0],
client.getRepos(), client.cfg))
self.troveTups = set()
for withFiles, trv in troveCache.cache.values():
for nvf in trv.iterTroveList(strongRefs=True, weakRefs=True):
self.troveTups.add(nvf)
def mirrorTroveInfo(src, targets, mark, cfg, resync=False):
if resync:
log.debug("performing a full trove info sync")
infoList = _getAllInfo(src, cfg)
infoList = [(mark, t, ti) for t, ti in infoList ]
else:
log.debug("getting new trove info entries")
infoList = _getNewInfo(src, cfg, mark)
log.debug("obtained %d trove info records for mirroring", len(infoList))
infoList = [(m,t,ti) for (m,t,ti) in infoList if _filterTup(t, cfg)]
if not len(infoList):
log.debug("no troveinfo records need to be mirrored")
return 0
log.debug("mirroring %d changed trove info records" % len(infoList))
updateCount = sum(_parallel(targets,
TargetRepository.setTroveInfo, infoList))
return updateCount
def _copyDirs(self):
for (sourceDir, targetDir) in self.dirsToCopy:
targetDir = self.root + targetDir
if os.path.exists(targetDir):
if os.path.islink(targetDir):
os.unlink(targetDir)
else:
util.rmtree(targetDir)
util.mkdirChain(os.path.realpath(os.path.dirname(targetDir)))
log.debug("copying dir %s into chroot:%s", sourceDir, targetDir)
try:
shutil.copytree(sourceDir, os.path.realpath(targetDir))
except shutil.Error, e:
errorList = '\n'.join('cannot copy %s to %s: %s' % x
for x in e.args[0])
raise errors.OpenError('Could not copy in directory %s:\n%s' %
(sourceDir, errorList))
def checkSyncRepos(config, sourceRepos, targetRepos):
checkConfig(config)
targets = _makeTargets(config, targetRepos)
log.setVerbosity(log.DEBUG)
# retrieve the set of troves from a give repository
def _getTroveSet(config, repo):
def _flatten(troveSpec):
l = []
for name, versionD in troveSpec.iteritems():
for version, flavorList in versionD.iteritems():
l += [(name, version, flavor) for flavor in flavorList]
return set(l)
troveSpecs = {}
if config.labels:
d = troveSpecs.setdefault(None, {})
for l in config.labels:
d[l] = ''
t = repo.getTroveVersionsByLabel(
troveSpecs, troveTypes=netclient.TROVE_QUERY_ALL)
else:
troveSpecs = {None: None}
t = repo.getTroveVersionList(config.host,
troveSpecs,
troveTypes=netclient.TROVE_QUERY_ALL)
return _flatten(t)
# compare source with each target
def _compare(src, dst):
srcName, srcSet = src
dstName, dstSet = dst
counter = 0
for x in srcSet.difference(dstSet):
log.debug(" - %s %s " % (srcName, x))
counter += 1
for x in dstSet.difference(srcSet):
log.debug(" + %s %s" % (dstName, x))
counter += 1
return counter
log.debug("Retrieving list of troves from source %s" %
str(sourceRepos.c.map))
sourceSet = _getTroveSet(config, sourceRepos)
hasDiff = 0
for target in targets:
log.debug("Retrieving list of troves from %s %s" %
(target.name, str(target.repo.c.map)))
targetSet = _getTroveSet(config, target.repo)
log.debug("Diffing source and %s" % target.name)
hasDiff += _compare(("source", sourceSet), (target.name, targetSet))
log.debug("Done")
return hasDiff
def _exceptionOccured(self, exc_info):
etype, e, tb = exc_info
# format the exception
msg = "%s" % etype.__name__
s = str(e)
if s:
msg += ": %s" % s
# get the line info that raised the exception
inner = tb.tb_next
while inner.tb_next:
inner = inner.tb_next
filename = inner.tb_frame.f_code.co_filename
linenum = inner.tb_frame.f_lineno
log.warning("Unhandled exception occurred when invoking callback:\n" "%s:%s\n" " %s", filename, linenum, msg)
# log the full traceback if debugging (--debug=all)
log.debug("".join(traceback.format_exception(*exc_info)))
if not hasattr(self, "exceptions"):
self.exceptions = []
self.exceptions.append(e)
def setTroveInfo(self, infoList):
log.debug("%s checking what troveinfo needs to be mirrored", self.name)
# Items whose mark is the same as currentMark might not have their trove
# available on the server (it might be coming as part of this mirror
# run).
inQuestion = [ x[1] for x in infoList if str(long(x[0])) >= self.mark ]
present = self.repo.hasTroves(inQuestion, hidden=True)
# filter out the not present troves which will get mirrored in
# the current mirror run
infoList = [ (t, ti) for (m, t, ti) in infoList if present.get(t, True) ]
# avoid busy work for troveinfos which are empty
infoList = [ (t, ti) for (t, ti) in infoList if len(ti.freeze()) > 0 ]
if self.test:
return 0
try:
self.repo.setTroveInfo(infoList)
except errors.InvalidServerVersion: # to older servers we can only transport sigs
infoList = [ (t, ti.sigs.freeze()) for t, ti in infoList ]
# only send up the troves that actually have a signature change
infoList = [ x for x in infoList if len(x[1]) > 0 ]
log.debug("%s pushing %d trove sigs...", self.name, len(infoList))
self.repo.setTroveSigs(infoList)
else:
log.debug("%s uploaded %d info records", self.name, len(infoList))
return len(infoList)
def getTrovesByPath(repos, pathList, versionFilter, flavorFilter, labelPath,
defaultFlavor):
if not pathList:
return []
if versionFilter == VERSION_FILTER_ALL:
queryFn = repos.getTroveVersionsByPath
elif versionFilter == VERSION_FILTER_LEAVES:
queryFn = repos.getTroveLeavesByPath
elif versionFilter == VERSION_FILTER_LATEST:
queryFn = repos.getTroveLeavesByPath
else:
assert (0)
allResults = {}
for label in labelPath:
try:
results = queryFn(pathList, label)
except errors.MethodNotSupported:
log.debug('repository server for the %s label does not support '
'queries by path' % label)
continue
for path, tups in results.iteritems():
allResults.setdefault(path, []).extend(tups)
allResults = [allResults.get(x) for x in pathList]
finalList = []
for tupList in allResults:
if not tupList:
continue
source = trovesource.SimpleTroveSource(tupList)
source.searchAsRepository()
troveNames = set(x[0] for x in tupList)
# no affinity when searching by path.
results = getTrovesToDisplay(source, troveNames, [], [], versionFilter,
flavorFilter, labelPath, defaultFlavor,
None)
finalList.extend(results)
return finalList
def getTrovesByPath(repos, pathList, versionFilter, flavorFilter, labelPath,
defaultFlavor):
if not pathList:
return []
if versionFilter == VERSION_FILTER_ALL:
queryFn = repos.getTroveVersionsByPath
elif versionFilter == VERSION_FILTER_LEAVES:
queryFn = repos.getTroveLeavesByPath
elif versionFilter == VERSION_FILTER_LATEST:
queryFn = repos.getTroveLeavesByPath
else:
assert(0)
allResults = {}
for label in labelPath:
try:
results = queryFn(pathList, label)
except errors.MethodNotSupported:
log.debug('repository server for the %s label does not support '
'queries by path' %label)
continue
for path, tups in results.iteritems():
allResults.setdefault(path, []).extend(tups)
allResults = [ allResults.get(x) for x in pathList ]
finalList = [ ]
for tupList in allResults:
if not tupList:
continue
source = trovesource.SimpleTroveSource(tupList)
source.searchAsRepository()
troveNames = set(x[0] for x in tupList)
# no affinity when searching by path.
results = getTrovesToDisplay(source, troveNames, [], [],
versionFilter, flavorFilter, labelPath,
defaultFlavor, None)
finalList.extend(results)
return finalList
def setTroveInfo(self, infoList):
log.debug("%s checking what troveinfo needs to be mirrored", self.name)
# Items whose mark is the same as currentMark might not have their trove
# available on the server (it might be coming as part of this mirror
# run).
inQuestion = [x[1] for x in infoList if str(long(x[0])) >= self.mark]
present = self.repo.hasTroves(inQuestion, hidden=True)
# filter out the not present troves which will get mirrored in
# the current mirror run
infoList = [(t, ti) for (m, t, ti) in infoList if present.get(t, True)]
# avoid busy work for troveinfos which are empty
infoList = [(t, ti) for (t, ti) in infoList if len(ti.freeze()) > 0]
if self.test:
return 0
try:
self.repo.setTroveInfo(infoList)
except errors.InvalidServerVersion: # to older servers we can only transport sigs
infoList = [(t, ti.sigs.freeze()) for t, ti in infoList]
# only send up the troves that actually have a signature change
infoList = [x for x in infoList if len(x[1]) > 0]
log.debug("%s pushing %d trove sigs...", self.name, len(infoList))
self.repo.setTroveSigs(infoList)
else:
log.debug("%s uploaded %d info records", self.name, len(infoList))
return len(infoList)
def parseUpdateList(updateList,
keepExisting,
updateByDefault=True,
withFrozenFlavor=False):
# If keepExisting is true, we want our specifications to be relative
# to nothing. If it's false, they should be absolute as updateChangeSet
# interperts absolute jobs as ones which should be rooted (if there is
# anything available to root them to).
areAbsolute = not keepExisting
applyList = []
if type(updateList) is str:
updateList = (updateList, )
for updateStr in updateList:
if os.path.exists(updateStr) and os.path.isfile(updateStr):
applyList.append(_getChangeSet(updateStr))
continue
else:
troveSpec = parseTroveSpec(updateStr,
withFrozenFlavor=withFrozenFlavor)
if troveSpec[0][0] == '-':
applyList.append(
(troveSpec[0], troveSpec[1:], (None, None), False))
elif troveSpec[0][0] == '+':
applyList.append(
(troveSpec[0], (None, None), troveSpec[1:], areAbsolute))
elif updateByDefault:
applyList.append(
(troveSpec[0], (None, None), troveSpec[1:], areAbsolute))
else:
applyList.append(
(troveSpec[0], troveSpec[1:], (None, None), False))
log.debug("will look for %s", applyList[-1])
# dedup
return set(applyList)
def _exceptionOccured(self, exc_info):
etype, e, tb = exc_info
# format the exception
msg = '%s' % etype.__name__
s = str(e)
if s:
msg += ': %s' % s
# get the line info that raised the exception
inner = tb.tb_next
while inner.tb_next:
inner = inner.tb_next
filename = inner.tb_frame.f_code.co_filename
linenum = inner.tb_frame.f_lineno
log.warning(
'Unhandled exception occurred when invoking callback:\n'
'%s:%s\n'
' %s', filename, linenum, msg)
# log the full traceback if debugging (--debug=all)
log.debug(''.join(traceback.format_exception(*exc_info)))
if not hasattr(self, 'exceptions'):
self.exceptions = []
self.exceptions.append(e)
def _getAllInfo(src, cfg):
log.debug("resync all trove info from source. This will take a while...")
# grab the full list of all the trove versions and flavors in the src
troveDict = src.getTroveVersionList(cfg.host, { None : None })
troveList = []
# filter out the stuff we don't need
for name, versionD in troveDict.iteritems():
for version, flavorList in versionD.iteritems():
for flavor in flavorList:
tup = (name, version, flavor)
troveList.append(tup)
del troveDict
# retrieve the sigs and the metadata records to sync over
sigList = src.getTroveSigs(troveList)
metaList = src.getTroveInfo(trove._TROVEINFO_TAG_METADATA, troveList)
infoList = []
for t, s, ti in itertools.izip(troveList, sigList, metaList):
if ti is None:
ti = trove.TroveInfo()
ti.sigs.thaw(s)
infoList.append((t, ti))
return infoList
def _getAllInfo(src, cfg):
log.debug("resync all trove info from source. This will take a while...")
# grab the full list of all the trove versions and flavors in the src
troveDict = src.getTroveVersionList(cfg.host, {None: None})
troveList = []
# filter out the stuff we don't need
for name, versionD in troveDict.iteritems():
for version, flavorList in versionD.iteritems():
for flavor in flavorList:
tup = (name, version, flavor)
troveList.append(tup)
del troveDict
# retrieve the sigs and the metadata records to sync over
sigList = src.getTroveSigs(troveList)
metaList = src.getTroveInfo(trove._TROVEINFO_TAG_METADATA, troveList)
infoList = []
for t, s, ti in itertools.izip(troveList, sigList, metaList):
if ti is None:
ti = trove.TroveInfo()
ti.sigs.thaw(s)
infoList.append((t, ti))
return infoList
def copytree(sources, dest, symlinks=False, filemode=None, dirmode=None, fileowner=None, dirowner=None, callback=None):
"""
Copies tree(s) from sources to dest, returning a list of
the filenames that it has written.
"""
sourcelist = []
totalFiles = 0
for source in util.braceGlob(sources):
if os.path.isdir(source):
if source[-1] == "/":
source = source[:-1]
thisdest = "%s%s%s" % (dest, os.sep, os.path.basename(source))
log.debug("copying [tree] %s to %s", source, thisdest)
_copytree(source, thisdest, symlinks, callback=callback)
if dirmode:
os.chmod(thisdest, dirmode)
if dirowner:
os.chown(thisdest, *dirowner)
os.path.walk(
source, _copyVisit, (sourcelist, len(source), thisdest, filemode, dirmode, fileowner, dirowner)
)
else:
log.debug("copying [file] %s to %s", source, dest)
shutil.copy2(source, dest)
totalFiles += 1
if callback:
callback(totalFiles)
if dest.endswith(os.sep):
thisdest = dest + os.sep + os.path.basename(source)
else:
thisdest = dest
if filemode:
os.chmod(thisdest, filemode)
if fileowner:
os.chown(thisdest, *fileowner)
sourcelist.append(thisdest)
return sourcelist
def splitJobList(jobList, src, targetSet, hidden = False, callback = ChangesetCallback()):
log.debug("Changeset Key conflict detected; splitting job further...")
jobs = {}
for job in jobList:
name = job[0]
if ':' in name:
name = name.split(':')[0]
l = jobs.setdefault(name, [])
l.append(job)
i = 0
for smallJobList in jobs.itervalues():
(outFd, tmpName) = util.mkstemp()
os.close(outFd)
log.debug("jobsplit %d of %d %s" % (
i + 1, len(jobs), displayBundle([(0,x) for x in smallJobList])))
src.createChangeSetFile(smallJobList, tmpName, recurse = False,
callback = callback, mirrorMode = True)
for target in targetSet:
target.commitChangeSetFile(tmpName, hidden = hidden, callback = callback)
os.unlink(tmpName)
callback.done()
i += 1
return
def recurseTrove(sourceRepos, name, version, flavor,
callback = ChangesetCallback()):
global recursedGroups
assert(trove.troveIsGroup(name))
# there's nothing much we can recurse from the source
if name.endswith(":source"):
return []
# avoid grabbing the same group multiple times
if (name, version, flavor) in recursedGroups:
return []
log.debug("recursing group trove: %s=%s[%s]" % (name, version, flavor))
groupCs = sourceRepos.createChangeSet(
[(name, (None, None), (version, flavor), True)],
withFiles=False, withFileContents=False, recurse=False,
callback = callback)
recursedGroups.add((name, version, flavor))
ret = []
for troveCs in groupCs.iterNewTroveList():
for name, ops in troveCs.iterChangedTroves(True, True):
for oper, version, flavor, byDefault in ops:
if oper != '-':
ret.append((name, version, flavor))
return ret
def parseUpdateList(updateList, keepExisting, updateByDefault=True,
withFrozenFlavor=False):
# If keepExisting is true, we want our specifications to be relative
# to nothing. If it's false, they should be absolute as updateChangeSet
# interperts absolute jobs as ones which should be rooted (if there is
# anything available to root them to).
areAbsolute = not keepExisting
applyList = []
if type(updateList) is str:
updateList = ( updateList, )
for updateStr in updateList:
if os.path.exists(updateStr) and os.path.isfile(updateStr):
applyList.append(_getChangeSet(updateStr))
continue
else:
troveSpec = parseTroveSpec(updateStr,
withFrozenFlavor=withFrozenFlavor)
if troveSpec[0][0] == '-':
applyList.append((troveSpec[0], troveSpec[1:],
(None, None), False))
elif troveSpec[0][0] == '+':
applyList.append((troveSpec[0], (None, None),
troveSpec[1:], areAbsolute))
elif updateByDefault:
applyList.append((troveSpec[0], (None, None),
troveSpec[1:], areAbsolute))
else:
applyList.append((troveSpec[0], troveSpec[1:],
(None, None), False))
log.debug("will look for %s", applyList[-1])
# dedup
return set(applyList)
def checkSyncRepos(config, sourceRepos, targetRepos):
checkConfig(config)
targets = _makeTargets(config, targetRepos)
log.setVerbosity(log.DEBUG)
# retrieve the set of troves from a give repository
def _getTroveSet(config, repo):
def _flatten(troveSpec):
l = []
for name, versionD in troveSpec.iteritems():
for version, flavorList in versionD.iteritems():
l += [ (name, version, flavor) for flavor in flavorList ]
return set(l)
troveSpecs = {}
if config.labels:
d = troveSpecs.setdefault(None, {})
for l in config.labels:
d[l] = ''
t = repo.getTroveVersionsByLabel(troveSpecs, troveTypes = netclient.TROVE_QUERY_ALL)
else:
troveSpecs = {None : None}
t = repo.getTroveVersionList(config.host, troveSpecs,
troveTypes = netclient.TROVE_QUERY_ALL)
return _flatten(t)
# compare source with each target
def _compare(src, dst):
srcName, srcSet = src
dstName, dstSet = dst
counter = 0
for x in srcSet.difference(dstSet):
log.debug(" - %s %s " % (srcName, x))
counter += 1
for x in dstSet.difference(srcSet):
log.debug(" + %s %s" % (dstName, x))
counter += 1
return counter
log.debug("Retrieving list of troves from source %s" % str(sourceRepos.c.map))
sourceSet = _getTroveSet(config, sourceRepos)
hasDiff = 0
for target in targets:
log.debug("Retrieving list of troves from %s %s" % (target.name, str(target.repo.c.map)))
targetSet = _getTroveSet(config, target.repo)
log.debug("Diffing source and %s" % target.name)
hasDiff += _compare( ("source", sourceSet), (target.name, targetSet) )
log.debug("Done")
return hasDiff
def _getNewSigs(src, cfg, mark):
# talking to an old source server. We do the best and we get the sigs out
sigList = src.getNewSigList(cfg.host, str(mark))
log.debug("obtained %d changed trove sigs", len(sigList))
sigList = [ x for x in sigList if _filterTup(x[1], cfg) ]
log.debug("%d changed sigs after label and match filtering", len(sigList))
# protection against duplicate items returned in the list by some servers
sigList = list(set(sigList))
sigList.sort(lambda a,b: cmp(a[0], b[0]))
log.debug("downloading %d signatures from source repository", len(sigList))
# XXX: we could also get the metadata in here, but getTroveInfo
# would use a getChangeSet call against older repos, severely
# impacting performance
sigs = src.getTroveSigs([ x[1] for x in sigList ])
# need to convert the sigs into TroveInfo instances
def _sig2info(sig):
ti = trove.TroveInfo()
ti.sigs.thaw(sig)
return ti
sigs = [ _sig2info(s) for s in sigs]
# we're gonna iterate repeatedely over the returned set, no itertools can do
return [(m, t, ti) for (m,t),ti in itertools.izip(sigList, sigs) ]
def _getNewSigs(src, cfg, mark):
# talking to an old source server. We do the best and we get the sigs out
sigList = src.getNewSigList(cfg.host, str(mark))
log.debug("obtained %d changed trove sigs", len(sigList))
sigList = [ x for x in sigList if _filterTup(x[1], cfg) ]
log.debug("%d changed sigs after label and match filtering", len(sigList))
# protection against duplicate items returned in the list by some servers
sigList = list(set(sigList))
sigList.sort(lambda a,b: cmp(a[0], b[0]))
log.debug("downloading %d signatures from source repository", len(sigList))
# XXX: we could also get the metadata in here, but getTroveInfo
# would use a getChangeSet call against older repos, severely
# impacting performance
sigs = src.getTroveSigs([ x[1] for x in sigList ])
# need to convert the sigs into TroveInfo instances
def _sig2info(sig):
ti = trove.TroveInfo()
ti.sigs.thaw(sig)
return ti
sigs = [ _sig2info(s) for s in sigs]
# we're gonna iterate repeatedely over the returned set, no itertools can do
return [(m, t, ti) for (m,t),ti in itertools.izip(sigList, sigs) ]
def mirrorRepository(
sourceRepos,
targetRepos,
cfg,
test=False,
sync=False,
syncSigs=False,
callback=ChangesetCallback(),
fastSync=False,
referenceRepos=None,
):
if referenceRepos is None:
referenceRepos = sourceRepos
checkConfig(cfg)
targets = _makeTargets(cfg, targetRepos, test)
log.debug("-" * 20 + " start loop " + "-" * 20)
hidden = len(targets) > 1 or cfg.useHiddenCommits
if hidden:
log.debug("will use hidden commits to synchronize target mirrors")
if sync:
currentMark = -1
else:
marks = [t.getMirrorMark() for t in targets]
# we use the oldest mark as a starting point (since we have to
# get stuff from source for that oldest one anyway)
currentMark = min(marks)
log.debug("using common mirror mark %s", currentMark)
# reset mirror mark to the lowest common denominator
for t in targets:
if t.getMirrorMark() != currentMark:
t.setMirrorMark(currentMark)
# mirror gpg signatures from the src into the targets
for t in targets:
t.mirrorGPG(referenceRepos, cfg.host)
# mirror changed trove information for troves already mirrored
if fastSync:
updateCount = 0
log.debug("skip trove info records sync because of fast-sync")
else:
updateCount = mirrorTroveInfo(referenceRepos, targets, currentMark,
cfg, syncSigs)
newMark, troveList = getTroveList(referenceRepos, cfg, currentMark)
if not troveList:
if newMark > currentMark: # something was returned, but filtered out
for t in targets:
t.setMirrorMark(newMark)
return -1 # call again
return 0
# prepare a new max mark to be used when we need to break out of a loop
crtMaxMark = max(long(x[0]) for x in troveList)
if currentMark > 0 and crtMaxMark == currentMark:
# if we're hung on the current max then we need to
# forcibly advance the mark in case we're stuck
crtMaxMark += 1 # only used if we filter out all troves below
initTLlen = len(troveList)
# removed troves are a special blend - we keep them separate
removedSet = set(
[x[1] for x in troveList if x[2] == trove.TROVE_TYPE_REMOVED])
troveList = [(x[0], x[1]) for x in troveList
if x[2] != trove.TROVE_TYPE_REMOVED]
# figure out if we need to recurse the group-troves
if cfg.recurseGroups:
# avoid adding duplicates
troveSetList = set([x[1] for x in troveList])
for mark, (name, version, flavor) in troveList:
if trove.troveIsGroup(name):
recTroves = recurseTrove(referenceRepos,
name,
version,
flavor,
callback=callback)
# add sources here:
if cfg.includeSources:
troveInfo = referenceRepos.getTroveInfo(
trove._TROVEINFO_TAG_SOURCENAME, recTroves)
sourceComps = set()
for nvf, source in itertools.izip(recTroves, troveInfo):
sourceComps.add((source(), nvf[1].getSourceVersion(),
parseFlavor('')))
recTroves.extend(sourceComps)
# add the results at the end with the current mark
for (n, v, f) in recTroves:
if (n, v, f) not in troveSetList:
troveList.append((mark, (n, v, f)))
troveSetList.add((n, v, f))
log.debug("after group recursion %d troves are needed", len(troveList))
# we need to make sure we mirror the GPG keys of any newly added troves
newHosts = set(
[x[1].getHost() for x in troveSetList.union(removedSet)])
for host in newHosts.difference(set([cfg.host])):
for t in targets:
t.mirrorGPG(referenceRepos, host)
# we check which troves from the troveList are needed on each
# target and we split the troveList into separate lists depending
# on how many targets require each
byTarget = {}
targetSetList = []
if len(troveList):
byTrove = {}
for i, target in enumerate(targets):
for t in target.addTroveList(troveList):
bt = byTrove.setdefault(t, set())
bt.add(i)
# invert the dict by target now
for trv, ts in byTrove.iteritems():
targetSet = [targets[i] for i in ts]
try:
targetIdx = targetSetList.index(targetSet)
except ValueError:
targetSetList.append(targetSet)
targetIdx = len(targetSetList) - 1
bt = byTarget.setdefault(targetIdx, [])
bt.append(trv)
del byTrove
# if we were returned troves, but we filtered them all out, advance the
# mark and signal "try again"
if len(byTarget) == 0 and len(removedSet) == 0 and initTLlen:
# we had troves and now we don't
log.debug("no troves found for our label %s" % cfg.labels)
for t in targets:
t.setMirrorMark(crtMaxMark)
# try again
return -1
# now we get each section of the troveList for each targetSet. We
# start off mirroring by those required by fewer targets, using
# the assumption that those troves are what is required for the
# targets to catch up to a common set
if len(byTarget) > 1:
log.debug("split %d troves into %d chunks by target", len(troveList),
len(byTarget))
# sort the targetSets by length
targetSets = list(enumerate(targetSetList))
targetSets.sort(lambda a, b: cmp(len(a[1]), len(b[1])))
bundlesMark = 0
for idx, targetSet in targetSets:
troveList = byTarget[idx]
if not troveList: # XXX: should not happen...
continue
log.debug("mirroring %d troves into %d targets", len(troveList),
len(targetSet))
# since these troves are required for all targets, we can use
# the "first" one to build the relative changeset requests
target = list(targetSet)[0]
bundles = buildBundles(sourceRepos, target, troveList,
cfg.absoluteChangesets)
for i, bundle in enumerate(bundles):
jobList = [x[1] for x in bundle]
# XXX it's a shame we can't give a hint as to what server to use
# to avoid having to open the changeset and read in bits of it
if test:
log.debug("test mode: not mirroring (%d of %d) %s" %
(i + 1, len(bundles), jobList))
updateCount += len(bundle)
continue
(outFd, tmpName) = util.mkstemp()
os.close(outFd)
log.debug("getting (%d of %d) %s" %
(i + 1, len(bundles), displayBundle(bundle)))
try:
sourceRepos.createChangeSetFile(jobList,
tmpName,
recurse=False,
callback=callback,
mirrorMode=True)
except changeset.ChangeSetKeyConflictError:
splitJobList(jobList,
sourceRepos,
targetSet,
hidden=hidden,
callback=callback)
else:
for target in targetSet:
target.commitChangeSetFile(tmpName,
hidden=hidden,
callback=callback)
try:
os.unlink(tmpName)
except OSError:
pass
callback.done()
updateCount += len(bundle)
# compute the max mark of the bundles we comitted
mark = max([min([x[0] for x in bundle]) for bundle in bundles])
if mark > bundlesMark:
bundlesMark = mark
else: # only when we're all done looping advance mark to the new max
if bundlesMark == 0 or bundlesMark <= currentMark:
bundlesMark = crtMaxMark # avoid repeating the same query...
for target in targets:
if hidden: # if we've hidden the last commits, show them now
target.presentHiddenTroves()
target.setMirrorMark(bundlesMark)
# mirroring removed troves requires one by one processing
for target in targets:
copySet = removedSet.copy()
updateCount += mirrorRemoved(referenceRepos,
target.repo,
copySet,
test=test,
callback=callback)
# if this was a noop because the removed troves were already mirrored
# we need to keep going
if updateCount == 0 and len(removedSet):
for target in targets:
target.setMirrorMark(crtMaxMark)
return -1
return updateCount
def dbg(self, *args, **kwargs):
args = self._addClassName(args)
log.debug(*args, **kwargs)
def getTroveList(src, cfg, mark):
# FIXME: getNewTroveList should accept and only return troves on
# the labels we're interested in
log.debug("looking for new troves")
# make sure we always treat the mark as an integer
troveList = [(long(m), (n, v, f), t)
for m, (n, v,
f), t in src.getNewTroveList(cfg.host, str(mark))]
if not len(troveList):
# this should be the end - no more troves to look at
log.debug("no new troves found")
return (mark, [])
# we need to protect ourselves from duplicate items in the troveList
l = len(troveList)
troveList = list(set(troveList))
if len(troveList) < l:
l = len(troveList)
log.debug("after duplicate elimination %d troves are left",
len(troveList))
# if we filter out the entire list of troves we have been
# returned, we need to tell the caller what was the highest mark
# we had so it can continue asking for more
maxMark = max([x[0] for x in troveList])
# filter out troves on labels and parse through matchTroves
troveList = [x for x in troveList if _filterTup(x[1], cfg)]
if len(troveList) < l:
l = len(troveList)
log.debug("after label filtering and matchTroves %d troves are left",
l)
if not troveList:
return (maxMark, [])
# sort deterministically by mark, version, flavor, reverse name
troveList.sort(lambda a, b: cmp(a[0], b[0]) or cmp(a[1][1], b[1][1]) or
cmp(a[1][2], b[1][2]) or cmp(b[1][0], a[1][0]))
log.debug("%d new troves returned", len(troveList))
# We cut off the last troves that have the same flavor, version to
# avoid committing an incomplete trove. This could happen if the
# server side only listed some of a trove's components due to
# server side limits on how many results it can return on each query
lastIdx = len(troveList) - 1
# compare with the last one
ml, (nl, vl, fl), tl = troveList[-1]
while lastIdx >= 0:
lastIdx -= 1
m, (n, v, f), t = troveList[lastIdx]
if v == vl and f == fl:
continue
lastIdx += 1
break
# the min mark of the troves we skip has to be higher than max
# mark of troves we'll commit or otherwise we'll skip them for good...
if lastIdx >= 0:
firstMark = max([x[0] for x in troveList[:lastIdx]])
lastMark = min([x[0] for x in troveList[lastIdx:]])
if lastMark > firstMark:
troveList = troveList[:lastIdx]
log.debug("reduced new trove list to %d to avoid partial commits",
len(troveList))
# since we're returning at least on trove, the caller will make the next mark decision
return (mark, troveList)
def mirrorRepository(sourceRepos, targetRepos, cfg,
test = False, sync = False, syncSigs = False,
callback = ChangesetCallback(),
fastSync = False,
referenceRepos=None,
):
if referenceRepos is None:
referenceRepos = sourceRepos
checkConfig(cfg)
targets = _makeTargets(cfg, targetRepos, test)
log.debug("-" * 20 + " start loop " + "-" * 20)
hidden = len(targets) > 1 or cfg.useHiddenCommits
if hidden:
log.debug("will use hidden commits to synchronize target mirrors")
if sync:
currentMark = -1
else:
marks = [ t.getMirrorMark() for t in targets ]
# we use the oldest mark as a starting point (since we have to
# get stuff from source for that oldest one anyway)
currentMark = min(marks)
log.debug("using common mirror mark %s", currentMark)
# reset mirror mark to the lowest common denominator
for t in targets:
if t.getMirrorMark() != currentMark:
t.setMirrorMark(currentMark)
# mirror gpg signatures from the src into the targets
for t in targets:
t.mirrorGPG(referenceRepos, cfg.host)
# mirror changed trove information for troves already mirrored
if fastSync:
updateCount = 0
log.debug("skip trove info records sync because of fast-sync")
else:
updateCount = mirrorTroveInfo(referenceRepos, targets, currentMark,
cfg, syncSigs)
newMark, troveList = getTroveList(referenceRepos, cfg, currentMark)
if not troveList:
if newMark > currentMark: # something was returned, but filtered out
for t in targets:
t.setMirrorMark(newMark)
return -1 # call again
return 0
# prepare a new max mark to be used when we need to break out of a loop
crtMaxMark = max(long(x[0]) for x in troveList)
if currentMark > 0 and crtMaxMark == currentMark:
# if we're hung on the current max then we need to
# forcibly advance the mark in case we're stuck
crtMaxMark += 1 # only used if we filter out all troves below
initTLlen = len(troveList)
# removed troves are a special blend - we keep them separate
removedSet = set([ x[1] for x in troveList if x[2] == trove.TROVE_TYPE_REMOVED ])
troveList = [ (x[0], x[1]) for x in troveList if x[2] != trove.TROVE_TYPE_REMOVED ]
# figure out if we need to recurse the group-troves
if cfg.recurseGroups:
# avoid adding duplicates
troveSetList = set([x[1] for x in troveList])
for mark, (name, version, flavor) in troveList:
if trove.troveIsGroup(name):
recTroves = recurseTrove(referenceRepos, name,
version, flavor, callback=callback)
# add sources here:
if cfg.includeSources:
troveInfo = referenceRepos.getTroveInfo(
trove._TROVEINFO_TAG_SOURCENAME, recTroves)
sourceComps = set()
for nvf, source in itertools.izip(recTroves, troveInfo):
sourceComps.add((source(), nvf[1].getSourceVersion(),
parseFlavor('')))
recTroves.extend(sourceComps)
# add the results at the end with the current mark
for (n, v, f) in recTroves:
if (n, v, f) not in troveSetList:
troveList.append((mark, (n, v, f)))
troveSetList.add((n, v, f))
log.debug("after group recursion %d troves are needed", len(troveList))
# we need to make sure we mirror the GPG keys of any newly added troves
newHosts = set([x[1].getHost() for x in troveSetList.union(removedSet)])
for host in newHosts.difference(set([cfg.host])):
for t in targets:
t.mirrorGPG(referenceRepos, host)
# we check which troves from the troveList are needed on each
# target and we split the troveList into separate lists depending
# on how many targets require each
byTarget = {}
targetSetList = []
if len(troveList):
byTrove = {}
for i, target in enumerate(targets):
for t in target.addTroveList(troveList):
bt = byTrove.setdefault(t, set())
bt.add(i)
# invert the dict by target now
for trv, ts in byTrove.iteritems():
targetSet = [ targets[i] for i in ts ]
try:
targetIdx = targetSetList.index(targetSet)
except ValueError:
targetSetList.append(targetSet)
targetIdx = len(targetSetList)-1
bt = byTarget.setdefault(targetIdx, [])
bt.append(trv)
del byTrove
# if we were returned troves, but we filtered them all out, advance the
# mark and signal "try again"
if len(byTarget) == 0 and len(removedSet) == 0 and initTLlen:
# we had troves and now we don't
log.debug("no troves found for our label %s" % cfg.labels)
for t in targets:
t.setMirrorMark(crtMaxMark)
# try again
return -1
# now we get each section of the troveList for each targetSet. We
# start off mirroring by those required by fewer targets, using
# the assumption that those troves are what is required for the
# targets to catch up to a common set
if len(byTarget) > 1:
log.debug("split %d troves into %d chunks by target", len(troveList), len(byTarget))
# sort the targetSets by length
targetSets = list(enumerate(targetSetList))
targetSets.sort(lambda a,b: cmp(len(a[1]), len(b[1])))
bundlesMark = 0
for idx, targetSet in targetSets:
troveList = byTarget[idx]
if not troveList: # XXX: should not happen...
continue
log.debug("mirroring %d troves into %d targets", len(troveList), len(targetSet))
# since these troves are required for all targets, we can use
# the "first" one to build the relative changeset requests
target = list(targetSet)[0]
bundles = buildBundles(sourceRepos, target, troveList, cfg.absoluteChangesets)
for i, bundle in enumerate(bundles):
jobList = [ x[1] for x in bundle ]
# XXX it's a shame we can't give a hint as to what server to use
# to avoid having to open the changeset and read in bits of it
if test:
log.debug("test mode: not mirroring (%d of %d) %s" % (i + 1, len(bundles), jobList))
updateCount += len(bundle)
continue
(outFd, tmpName) = util.mkstemp()
os.close(outFd)
log.debug("getting (%d of %d) %s" % (i + 1, len(bundles), displayBundle(bundle)))
try:
sourceRepos.createChangeSetFile(jobList, tmpName, recurse = False,
callback = callback, mirrorMode = True)
except changeset.ChangeSetKeyConflictError:
splitJobList(jobList, sourceRepos, targetSet, hidden=hidden,
callback=callback)
else:
for target in targetSet:
target.commitChangeSetFile(tmpName, hidden=hidden, callback=callback)
try:
os.unlink(tmpName)
except OSError:
pass
callback.done()
updateCount += len(bundle)
# compute the max mark of the bundles we comitted
mark = max([min([x[0] for x in bundle]) for bundle in bundles])
if mark > bundlesMark:
bundlesMark = mark
else: # only when we're all done looping advance mark to the new max
if bundlesMark == 0 or bundlesMark <= currentMark:
bundlesMark = crtMaxMark # avoid repeating the same query...
for target in targets:
if hidden: # if we've hidden the last commits, show them now
target.presentHiddenTroves()
target.setMirrorMark(bundlesMark)
# mirroring removed troves requires one by one processing
for target in targets:
copySet = removedSet.copy()
updateCount += mirrorRemoved(referenceRepos, target.repo, copySet,
test=test, callback=callback)
# if this was a noop because the removed troves were already mirrored
# we need to keep going
if updateCount == 0 and len(removedSet):
for target in targets:
target.setMirrorMark(crtMaxMark)
return -1
return updateCount
def getTroveList(src, cfg, mark):
# FIXME: getNewTroveList should accept and only return troves on
# the labels we're interested in
log.debug("looking for new troves")
# make sure we always treat the mark as an integer
troveList = [(long(m), (n,v,f), t) for m, (n,v,f), t in
src.getNewTroveList(cfg.host, str(mark))]
if not len(troveList):
# this should be the end - no more troves to look at
log.debug("no new troves found")
return (mark, [])
# we need to protect ourselves from duplicate items in the troveList
l = len(troveList)
troveList = list(set(troveList))
if len(troveList) < l:
l = len(troveList)
log.debug("after duplicate elimination %d troves are left", len(troveList))
# if we filter out the entire list of troves we have been
# returned, we need to tell the caller what was the highest mark
# we had so it can continue asking for more
maxMark = max([x[0] for x in troveList])
# filter out troves on labels and parse through matchTroves
troveList = [ x for x in troveList if _filterTup(x[1],cfg) ]
if len(troveList) < l:
l = len(troveList)
log.debug("after label filtering and matchTroves %d troves are left", l)
if not troveList:
return (maxMark, [])
# sort deterministically by mark, version, flavor, reverse name
troveList.sort(lambda a,b: cmp(a[0], b[0]) or
cmp(a[1][1], b[1][1]) or
cmp(a[1][2], b[1][2]) or
cmp(b[1][0], a[1][0]) )
log.debug("%d new troves returned", len(troveList))
# We cut off the last troves that have the same flavor, version to
# avoid committing an incomplete trove. This could happen if the
# server side only listed some of a trove's components due to
# server side limits on how many results it can return on each query
lastIdx = len(troveList)-1
# compare with the last one
ml, (nl,vl,fl), tl = troveList[-1]
while lastIdx >= 0:
lastIdx -= 1
m, (n,v,f), t = troveList[lastIdx]
if v == vl and f == fl:
continue
lastIdx += 1
break
# the min mark of the troves we skip has to be higher than max
# mark of troves we'll commit or otherwise we'll skip them for good...
if lastIdx >= 0:
firstMark = max([x[0] for x in troveList[:lastIdx]])
lastMark = min([x[0] for x in troveList[lastIdx:]])
if lastMark > firstMark:
troveList = troveList[:lastIdx]
log.debug("reduced new trove list to %d to avoid partial commits", len(troveList))
# since we're returning at least on trove, the caller will make the next mark decision
return (mark, troveList)
def presentHiddenTroves(self, newMark):
log.debug("%s unhiding comitted troves", self.name)
self.repo.presentHiddenTroves(self.cfg.host)
self.setMirrorMark(newMark)
def presentHiddenTroves(self):
log.debug("%s unhiding comitted troves", self.name)
self.repo.presentHiddenTroves(self.cfg.host)
return trovetup.TroveSpec(specStr,
allowEmptyName=allowEmptyName, withFrozenFlavor=withFrozenFlavor)
def _getChangeSet(path):
try:
cs = changeset.ChangeSetFromFile(path)
except BadContainer, msg:
# ensure that it is obvious that a file is being referenced
if path[0] not in './':
path = './' + path
log.error("'%s' is not a valid conary changeset: %s" %
(path, msg))
# XXX sys.exit is gross
import sys
sys.exit(1)
log.debug("found changeset file %s" % path)
return cs
def parseUpdateList(updateList, keepExisting, updateByDefault=True,
withFrozenFlavor=False):
# If keepExisting is true, we want our specifications to be relative
# to nothing. If it's false, they should be absolute as updateChangeSet
# interperts absolute jobs as ones which should be rooted (if there is
# anything available to root them to).
areAbsolute = not keepExisting
applyList = []
if type(updateList) is str:
updateList = ( updateList, )
def filterSuggestions(self, depList, sugg, suggMap):
"""
Given a list of several suggestions for one dependency,
pick the dep that matches the best.
"""
troves = set()
for (troveTup, depSet) in depList:
choicesBySolution = {}
seen = set()
if depSet in sugg:
suggList = set()
choicesAndDep = itertools.izip(sugg[depSet],
depSet.iterDeps(sort=True))
for choiceList, (depClass, dep) in choicesAndDep:
troveNames = set(x[0] for x in choiceList)
if self.db:
affTroveDict = \
dict((x, self.db.trovesByName(x))
for x in troveNames)
else:
affTroveDict = dict.fromkeys(troveNames, {})
# iterate over flavorpath -- use suggestions
# from first flavor on flavorpath that gets a match
for installFlavor in self.flavor:
choice = self.selectResolutionTrove(troveTup, dep,
depClass,
choiceList,
installFlavor,
affTroveDict)
if choice:
suggList.add(choice)
l = suggMap.setdefault(troveTup, set())
l.add(choice)
if choice not in seen:
if choice not in choicesBySolution:
d = deps.DependencySet()
choicesBySolution[choice] = d
else:
d = choicesBySolution[choice]
d.addDep(depClass, dep)
break
if choicesBySolution:
for choice, depSet in sorted(choicesBySolution.iteritems()):
seen.add(choice)
depSet = str(depSet).split('\n')
if len(depSet) > 5:
depSet = depSet[0:5] + ['...']
depSet = '\n '.join(depSet)
log.debug('Resolved:\n'
' %s=%s/%s[%s]\n'
' Required: %s\n'
' Adding: %s=%s/%s[%s]',
troveTup[0], troveTup[1].trailingLabel(), troveTup[1].trailingRevision(),troveTup[2], depSet, choice[0], choice[1].trailingLabel(), choice[1].trailingRevision(), choice[2])
troves.update([ (x[0], (None, None), x[1:], True)
for x in suggList ])
return troves
def setMirrorMark(self, mark):
self.mark = str(long(mark))
log.debug("%s setting mirror mark to %s", self.name, self.mark)
if self.test:
return
self.repo.setMirrorMark(self.cfg.host, self.mark)
def setMirrorMark(self, mark):
self.mark = str(long(mark))
log.debug("%s setting mirror mark to %s", self.name, self.mark)
if self.test:
return
self.repo.setMirrorMark(self.cfg.host, self.mark)
def repair(self):
if not self._status:
log.debug("no errors detected on check run, nothing to fix")
return True
return self.fix()
def presentHiddenTroves(self):
log.debug("%s unhiding comitted troves", self.name)
self.repo.presentHiddenTroves(self.cfg.host)
