def run(self):
log.info("Running: %s", self.__doc__)
ret = self.check()
if self._alwaysfix or (not ret and self._fix):
ret = self.repair()
log.info("%-7s: %s\n", ["FAIL", "Success"][int(bool(ret))], self.__doc__)
return ret
def makeFSImage(self, sizes):
root = self.workDir + "/root"
try:
# create an image file per mount point
imgFiles = {}
for mountPoint, req in self.mountDict.items():
size = sizes[mountPoint]
tag = mountPoint.replace("/", "")
tag = tag and tag or "root"
imgFiles[mountPoint] = path = self.mntPointFileName(mountPoint)
log.info("Creating mount point %s at %s with size %d bytes", mountPoint, path, size)
fs = self.makeBlankFS(path, req.fstype, size, fsLabel=req.name)
self.addFilesystem(mountPoint, fs)
self.mountAll()
# Install image contents.
self.installFileTree(root)
finally:
try:
self.umountAll()
util.rmtree(root, ignore_errors=True)
except:
log.logger.exception("Error unmounting partitions:")
return imgFiles
def searchNetworkSources(self, url, headers, single):
if url.scheme not in NETWORK_SCHEMES:
return
# check for negative cache entries to avoid spamming servers
if not single:
negativePath = self.repCache.checkNegativeCache(
self.recipeName, url)
if negativePath:
log.warning('not fetching %s (negative cache entry %s exists)',
url, negativePath)
return
log.info('Trying %s...', str(url))
if headers is None:
headers = {}
inFile = self._fetchUrl(url, headers)
if inFile is None:
self.repCache.createNegativeCacheEntry(self.recipeName, url)
else:
contentLength = int(inFile.headers.get('Content-Length', 0))
path = self.repCache.addFileToCache(self.recipeName, url, inFile,
contentLength)
if path:
raise PathFound(path, False)
return
def _fetchUrl(self, url, headers):
if isinstance(url, str):
url = laUrl(url)
retries = 3
if self.cfg.proxy and not self.noproxyFilter.bypassProxy(url.host):
retries = 7
inFile = None
for i in range(retries):
try:
# set up a handler that tracks cookies to handle
# sites like Colabnet that want to set a session cookie
cj = cookielib.LWPCookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
# add password handler if needed
if url.user:
url.passwd = url.passwd or ''
opener.add_handler(
HTTPBasicAuthHandler(url.user, url.passwd))
# add proxy and proxy password handler if needed
if self.cfg.proxy and \
not self.noproxyFilter.bypassProxy(url.host):
proxyPasswdMgr = urllib2.HTTPPasswordMgr()
for v in self.cfg.proxy.values():
pUrl = laUrl(v[1])
if pUrl.user:
pUrl.passwd = pUrl.passwd or ''
proxyPasswdMgr.add_password(
None, pUrl.asStr(noAuth=True, quoted=True),
url.user, url.passwd)
opener.add_handler(
urllib2.ProxyBasicAuthHandler(proxyPasswdMgr))
opener.add_handler(
urllib2.ProxyHandler(self.cfg.proxy))
if url.scheme == 'ftp':
urlStr = url.asStr(noAuth=False, quoted=True)
else:
urlStr = url.asStr(noAuth=True, quoted=True)
req = urllib2.Request(urlStr, headers=headers)
inFile = opener.open(req)
if not urlStr.startswith('ftp://'):
content_type = inFile.info().get('content-type')
if not url.explicit() and 'text/html' in content_type:
raise urllib2.URLError('"%s" not found' % urlStr)
log.info('Downloading %s...', urlStr)
break
except urllib2.HTTPError, msg:
if msg.code == 404:
return None
else:
log.error('error downloading %s: %s',
urlStr, str(msg))
return None
except urllib2.URLError:
return None
def searchNetworkSources(self, url, headers):
if url.scheme not in NETWORK_SCHEMES:
return
# check for negative cache entries to avoid spamming servers
negativePath = self.repCache.checkNegativeCache(self.recipeName, url)
if negativePath:
log.warning('not fetching %s (negative cache entry %s exists)',
url, negativePath)
return
log.info('Trying %s...', str(url))
if headers is None:
headers = {}
inFile = self._fetchUrl(url, headers)
if inFile is None:
self.repCache.createNegativeCacheEntry(self.recipeName, url)
else:
contentLength = int(inFile.headers.get('Content-Length', 0))
path = self.repCache.addFileToCache(self.recipeName, url,
inFile, contentLength)
if path:
raise PathFound(path, False)
return
def _getPathList(repos, cfg, recipePath, relative=False):
loader, recipeClass, sourceVersion = cook.getRecipeInfoFromPath(repos, cfg,
recipePath)
log.info("Getting relevant path information from %s..." % recipeClass.name)
recipeDir = os.path.dirname(recipePath)
srcdirs = [ recipeDir ]
recipeObj = None
buildLabel = sourceVersion.trailingLabel()
macros = {'buildlabel' : buildLabel.asString(),
'buildbranch' : sourceVersion.branch().asString()}
if recipe.isPackageRecipe(recipeClass):
recipeObj = recipeClass(cfg, None,
srcdirs, macros, lightInstance=True)
elif recipe.isGroupRecipe(recipeClass):
recipeObj = recipeClass(repos, cfg, buildLabel, None, None,
srcdirs=srcdirs,
extraMacros=macros)
else:
# no included files for the rest of the recipe types
pathList = [recipePath]
recipeObj = None
if recipeObj:
try:
if hasattr(recipeObj, 'loadPolicy'):
recipeObj.loadPolicy()
cook._callSetup(cfg, recipeObj)
except (conaryerrors.ConaryError, conaryerrors.CvcError), msg:
raise errors.RmakeError("could not initialize recipe: %s" % (msg))
pathList = recipeObj.fetchLocalSources() + [recipePath ]
def getTimestamps(self, troveTupList):
# look in the dep cache and trove cache
result = [ None ] * len(troveTupList)
for i, tup in enumerate(troveTupList):
result[i] = self.timeStampCache.get(tup[0:2])
if result[i] is None and self.troveIsCached(tup):
trv = self.cache[tup]
result[i] = trv.getVersion()
needed = [ (i, troveTup) for i, (troveTup, depSet) in
enumerate(izip(troveTupList, result))
if depSet is None ]
if not needed:
return result
# use the timeStamps call; it raises an error if it needs
# to access some repositories which don't support it
log.info("Getting timeStamps for %d troves" % len(needed))
try:
depList = self.troveSource.getTimestamps(
[ x[1] for x in needed ])
except netclient.PartialResultsError, e:
# we can't use this call everywhere; handle what we can and we'll
# deal with the None's later
depList = e.partialResults
def getTimestamps(self, troveTupList):
# look in the dep cache and trove cache
result = [None] * len(troveTupList)
for i, tup in enumerate(troveTupList):
result[i] = self.timeStampCache.get(tup[0:2])
if result[i] is None and self.troveIsCached(tup):
trv = self.cache[tup]
result[i] = trv.getVersion()
needed = [(i, troveTup)
for i, (troveTup,
depSet) in enumerate(izip(troveTupList, result))
if depSet is None]
if not needed:
return result
# use the timeStamps call; it raises an error if it needs
# to access some repositories which don't support it
log.info("Getting timeStamps for %d troves" % len(needed))
try:
depList = self.troveSource.getTimestamps([x[1] for x in needed])
except netclient.PartialResultsError, e:
# we can't use this call everywhere; handle what we can and we'll
# deal with the None's later
depList = e.partialResults
def updateRecipes(repos, cfg, recipeList, committedSources):
committedSourcesByNB = {}
for name, version, flavor in committedSources:
committedSourcesByNB[name, version.branch()] = version
for recipe in recipeList:
recipeDir = os.path.dirname(recipe)
stateFilePath = recipeDir + '/CONARY'
if not os.path.exists(stateFilePath):
continue
conaryStateFile = state.ConaryStateFromFile(stateFilePath)
if not conaryStateFile.hasSourceState():
continue
context = conaryStateFile.getContext()
stateFile = conaryStateFile.getSourceState()
troveName = stateFile.getName()
branch = stateFile.getBranch()
if (troveName, branch) not in committedSourcesByNB:
continue
stateVersion = stateFile.getVersion()
newVersion = committedSourcesByNB[troveName, branch]
if stateVersion != versions.NewVersion():
log.info('Updating %s after commit' % recipeDir)
try:
# Added in CNY-3035
checkin.nologUpdateSrc(repos, [recipeDir])
except checkin.builderrors.UpToDate:
pass # Don't mention if the source is already up to date
except checkin.builderrors.CheckinError, e:
e.logError()
except AttributeError:
checkin.updateSrc(repos, [recipeDir])
def stopBuild(results, pid, inF, csFile):
log.info('killing %s' % pid)
try:
os.kill(-pid, signal.SIGTERM)
except OSError, err:
if err.errno != errno.ESRCH:
raise
def stopBuild(results, pid, inF, csFile):
log.info('killing %s' % pid)
try:
os.kill(-pid, signal.SIGTERM)
except OSError, err:
if err.errno != errno.ESRCH:
raise
def _getPathList(repos, cfg, recipePath, relative=False):
loader, recipeClass, sourceVersion = cook.getRecipeInfoFromPath(repos, cfg,
recipePath)
log.info("Getting relevant path information from %s..." % recipeClass.name)
recipeDir = os.path.dirname(recipePath)
srcdirs = [ recipeDir ]
recipeObj = None
buildLabel = sourceVersion.trailingLabel()
macros = {'buildlabel' : buildLabel.asString(),
'buildbranch' : sourceVersion.branch().asString()}
if recipe.isPackageRecipe(recipeClass):
recipeObj = recipeClass(cfg, None,
srcdirs, macros, lightInstance=True)
elif recipe.isGroupRecipe(recipeClass):
recipeObj = recipeClass(repos, cfg, buildLabel, None, None,
srcdirs=srcdirs,
extraMacros=macros)
else:
# no included files for the rest of the recipe types
pathList = [recipePath]
recipeObj = None
if recipeObj:
try:
if hasattr(recipeObj, 'loadPolicy'):
recipeObj.loadPolicy()
cook._callSetup(cfg, recipeObj)
except (conaryerrors.ConaryError, conaryerrors.CvcError), msg:
raise errors.RmakeError("could not initialize recipe: %s" % (msg))
pathList = recipeObj.fetchLocalSources() + [recipePath ]
class CheckSchema(Checker):
""" checks for schema version """
def _postinit(self):
self._alwaysfix = self._fix
def check(self):
db = self.getDB()
dbVersion = db.getVersion()
if dbVersion.major == schema.VERSION.major:
log.info("schema is compatible with this codebase")
return True
log.error("codebase requires schema %s, repository has %s",
schema.VERSION, dbVersion)
return False
def fix(self):
db = self.getDB()
dbVersion = db.getVersion()
try:
log.info("performing a schema migration...")
newVersion = schema.loadSchema(db, doMigrate=True)
except sqlerrors.SchemaVersionError, e:
log.error(e.msg)
return False
if newVersion < dbVersion():
log.error("schema migration failed from %s to %s" %
(dbVersion, schema.VERSION))
return False
if newVersion == dbVersion: # did a big whoop noop
log.info("schema check complete")
else:
log.info("schema migration from %s to %s completed" %
(dbVersion, newVersion))
self.commit()
return True
def main():
opts = {}
opts["fix"] = options.NO_PARAM
cfg, opts, args = getServer(opts)
doFix = opts.has_key("fix")
if not args:
usage()
sys.exit(-1)
log.info("Starting tests\n")
ret = {}
all = False
if "ALL" in args:
all = True
# XXX: fixme - we should probably do something smarter and more automatic here...
if all or "schema" in args: # schema (migration) happens first
ret["schema"] = CheckSchema(cfg, doFix).run()
if all or "acls" in args:
ret["acls"] = CheckAcls(cfg, doFix).run()
if all or "latest" in args:
ret["latest"] = CheckLatest(cfg, doFix).run()
if all or "troveinfo" in args:
ret["troveinfo"] = CheckTroveInfo(cfg, doFix).run()
if all or "ctc" in args:
ret["ctc"] = CheckCTC(cfg, doFix).run()
if False in ret.values():
return False
return True
def main():
opts = {}
opts["fix"] = options.NO_PARAM
cfg, opts, args = getServer(opts)
doFix = opts.has_key("fix")
if not args:
usage()
sys.exit(-1)
log.info("Starting tests\n")
ret = {}
all = False
if "ALL" in args:
all = True
# XXX: fixme - we should probably do something smarter and more automatic here...
if all or "schema" in args: # schema (migration) happens first
ret["schema"] = CheckSchema(cfg, doFix).run()
if all or "acls" in args:
ret["acls"] = CheckAcls(cfg, doFix).run()
if all or "latest" in args:
ret["latest"] = CheckLatest(cfg, doFix).run()
if all or "troveinfo" in args:
ret["troveinfo"] = CheckTroveInfo(cfg, doFix).run()
if all or "ctc" in args:
ret["ctc"] = CheckCTC(cfg, doFix).run()
if False in ret.values():
return False
return True
def findAvailableTargetFlavors(self, repos):
if self.branchStr is None:
# redirect to nothing
return set()
if self.branchStr[0] == '/':
branch = versions.VersionFromString(self.branchStr)
if not isinstance(branch, versions.Branch):
raise builderrors.RecipeFileError, \
"Redirects must specify branches or labels, " \
"not versions"
log.info('redirecting to branches is deprecated; redirects must '
'be to labels')
matches = repos.getTroveLeavesByBranch(
{self.destName: {
branch: None
}})
else:
label = versions.Label(self.branchStr)
matches = repos.getTroveLatestByLabel(
{self.destName: {
label: None
}})
targetFlavors = set()
# Get the flavors and branch available on the target
for version, flavorList in matches.get(self.destName, {}).iteritems():
targetFlavors.update((version, x) for x in flavorList)
return targetFlavors
def _fetchUrl(self, url, headers):
if isinstance(url, str):
url = laUrl(url)
retries = 3
if self.cfg.proxy and not self.noproxyFilter.bypassProxy(url.host):
retries = 7
inFile = None
for i in range(retries):
try:
# set up a handler that tracks cookies to handle
# sites like Colabnet that want to set a session cookie
cj = cookielib.LWPCookieJar()
opener = urllib2.build_opener(urllib2.HTTPCookieProcessor(cj))
# add password handler if needed
if url.user:
url.passwd = url.passwd or ''
passwdMgr = self.BasicPasswordManager()
passwdMgr.add_password(url.user, url.passwd)
opener.add_handler(urllib2.HTTPBasicAuthHandler(passwdMgr))
# add proxy and proxy password handler if needed
if self.cfg.proxy and \
not self.noproxyFilter.bypassProxy(url.host):
proxyPasswdMgr = urllib2.HTTPPasswordMgr()
for v in self.cfg.proxy.values():
pUrl = laUrl(v[1])
if pUrl.user:
pUrl.passwd = pUrl.passwd or ''
proxyPasswdMgr.add_password(
None, pUrl.asStr(noAuth=True, quoted=True),
url.user, url.passwd)
opener.add_handler(
urllib2.ProxyBasicAuthHandler(proxyPasswdMgr))
opener.add_handler(urllib2.ProxyHandler(self.cfg.proxy))
if url.scheme == 'ftp':
urlStr = url.asStr(noAuth=False, quoted=True)
else:
urlStr = url.asStr(noAuth=True, quoted=True)
req = urllib2.Request(urlStr, headers=headers)
inFile = opener.open(req)
if not urlStr.startswith('ftp://'):
content_type = inFile.info()['content-type']
if not url.explicit() and 'text/html' in content_type:
raise urllib2.URLError('"%s" not found' % urlStr)
log.info('Downloading %s...', urlStr)
break
except urllib2.HTTPError, msg:
if msg.code == 404:
return None
else:
log.error('error downloading %s: %s', urlStr, str(msg))
return None
except urllib2.URLError:
return None
def run(self):
log.info("Running: %s", self.__doc__)
ret = self.check()
if self._alwaysfix or (not ret and self._fix):
ret = self.repair()
log.info("%-7s: %s\n", ["FAIL", "Success"][int(bool(ret))],
self.__doc__)
return ret
def startLogging():
import logging
# set a format which is simpler for console use
formatter = logging.Formatter('%(asctime)s %(message)s', datefmt = "%m-%d %H:%M")
# tell the handler to use this format
log.logger.handlers[0].setFormatter(formatter)
log.setVerbosity(log.DEBUG)
log.info("Logging system started")
def fix(self):
db = self.getDB()
cu = db.cursor()
log.info("adding missing entries to CheckTroveCache")
cu.executemany("insert into CheckTroveCache(patternId, itemId) values (?,?)",
((p,i) for (p,i) in self._status))
self.commit()
return self.check()
def check(self):
db = self.getDB()
dbVersion = db.getVersion()
if dbVersion.major == schema.VERSION.major:
log.info("schema is compatible with this codebase")
return True
log.error("codebase requires schema %s, repository has %s",
schema.VERSION, dbVersion)
return False
def fix(self):
db = self.getDB()
dbVersion = db.getVersion()
try:
log.info("performing a schema migration...")
newVersion = schema.loadSchema(db, doMigrate=True)
except sqlerrors.SchemaVersionError, e:
log.error(e.msg)
return False
def check(self):
db = self.getDB()
dbVersion = db.getVersion()
if dbVersion.major == schema.VERSION.major:
log.info("schema is compatible with this codebase")
return True
log.error("codebase requires schema %s, repository has %s",
schema.VERSION, dbVersion)
return False
def fix(self):
db = self.getDB()
cu = db.cursor()
log.info("adding missing entries to CheckTroveCache")
cu.executemany(
"insert into CheckTroveCache(patternId, itemId) values (?,?)",
((p, i) for (p, i) in self._status))
self.commit()
return self.check()
def startLogging():
import logging
# set a format which is simpler for console use
formatter = logging.Formatter('%(asctime)s %(message)s',
datefmt="%m-%d %H:%M")
# tell the handler to use this format
log.logger.handlers[0].setFormatter(formatter)
log.setVerbosity(log.DEBUG)
log.info("Logging system started")
def fix(self):
db = self.getDB()
dbVersion = db.getVersion()
try:
log.info("performing a schema migration...")
newVersion = schema.loadSchema(db, doMigrate=True)
except sqlerrors.SchemaVersionError, e:
log.error(e.msg)
return False
def fix(self):
from conary.repository.netrepos import versionops
db = self.getDB()
cu = db.cursor()
latest = versionops.LatestTable(db)
log.info("updating LatestCache table")
for itemId, branchId, flavorId in self._status:
latest.update(cu, itemId, branchId, flavorId)
log.info("update completed for LatestCache")
self.commit()
return True
def fix(self):
from conary.repository.netrepos import versionops
db = self.getDB()
cu = db.cursor()
latest = versionops.LatestTable(db)
log.info("updating LatestCache table")
for itemId, branchId, flavorId in self._status:
latest.update(cu, itemId, branchId, flavorId)
log.info("update completed for LatestCache")
self.commit()
return True
def fix(self):
db = self.getDB()
cu = db.cursor()
log.info("removing troveinfo records for non-prsent troves...")
schema.resetTable(cu, "tmpId")
cu.execute(""" insert into tmpId(id)
select distinct instanceId from Instances join TroveInfo using(instanceId)
where Instances.isPresent = ? """, instances.INSTANCE_PRESENT_MISSING)
cu.execute("delete from TroveInfo where instanceId in (select id from tmpId)")
self.commit()
return self.check()
def cookTrove(cfg, repos, logger, name, version, flavorList, targetLabel,
loadSpecsList=None, logData=None,
buildReqs=None, crossReqs=None):
if not isinstance(flavorList, (tuple, list)):
flavorList = [flavorList]
util.mkdirChain(cfg.root + '/tmp')
fd, csFile = tempfile.mkstemp(dir=cfg.root + '/tmp',
prefix='rmake-%s-' % name,
suffix='.ccs')
os.chmod(csFile, 0644)
os.close(fd)
logPath = cfg.root + '/tmp/rmake/%s-%s.log' % (name,
version.trailingRevision())
logFile = logfile.LogFile(logPath)
os.chmod(logPath, 0664)
os.chmod(cfg.root + '/tmp/rmake', 0775)
results = CookResults(name, version, flavorList)
# ignore child output problems
signal.signal(signal.SIGTTOU, signal.SIG_IGN)
inF, outF = pipereader.makePipes()
pid = os.fork()
if not pid:
try:
try:
signal.signal(signal.SIGTERM, signal.SIG_DFL)
os.close(inF)
os.setpgid(0, 0)
# don't accidentally make world writable files
os.umask(0022)
# don't allow us to create core dumps
resource.setrlimit(resource.RLIMIT_CORE, (0,0))
log.setVerbosity(log.DEBUG)
log.info("Cook process started (pid %s)" % os.getpid())
_cookTrove(cfg, repos, name, version, flavorList, targetLabel,
loadSpecsList,
csFile, buildReqs=buildReqs, crossReqs=crossReqs,
failureFd=outF, logger=logger)
except Exception, msg:
if len(flavorList) > 1:
errMsg = 'Error cooking %s=%s with flavors %s: %s' % \
(name, version, ', '.join([str(x) for x in flavorList]),
str(msg))
else:
errMsg = str(msg)
_buildFailed(outF, errMsg, traceback.format_exc())
logFile.close()
os._exit(1)
else:
logFile.close()
os._exit(0)
def check(self):
db = self.getDB()
cu = db.cursor()
log.info("checking for extraneous troveinfo records")
cu.execute(""" select instanceId, count(*)
from Instances join TroveInfo using(instanceId)
where Instances.isPresent = ?
group by instanceId having count(*) > 0 """, instances.INSTANCE_PRESENT_MISSING)
self._status = cu.fetchall()
if self._status:
log.warning("found %d non-present troves with troveinfo records", len(self._status))
return False
return True
def fix(self):
db = self.getDB()
cu = db.cursor()
log.info("removing troveinfo records for non-prsent troves...")
schema.resetTable(cu, "tmpId")
cu.execute(
""" insert into tmpId(id)
select distinct instanceId from Instances join TroveInfo using(instanceId)
where Instances.isPresent = ? """,
instances.INSTANCE_PRESENT_MISSING)
cu.execute(
"delete from TroveInfo where instanceId in (select id from tmpId)")
self.commit()
return self.check()
def getDepsForTroveList(self, troveTupList, provides = True,
requires = True):
def missingNeeded(depTuple):
if depTuple is None: return True
if provides and depTuple[0] is None: return True
if requires and depTuple[1] is None: return True
return False
def mergeCacheEntry(troveTup, depTuple):
existing = self.depCache.get(depTuple)
if existing is None:
self.depCache[troveTup] = depInfo
else:
self.depCache[troveTup] = (depTuple[0] or existing[0],
depTuple[1] or existing[1])
# look in the dep cache and trove cache
result = [ None ] * len(troveTupList)
for i, tup in enumerate(troveTupList):
result[i] = self.getDepCacheEntry(tup)
if result[i] is None and self.troveIsCached(tup):
trv = self.cache[tup]
result[i] = (trv.getProvides(), trv.getRequires())
elif result[i] is None and trove.troveIsPackage(tup[0]):
# packages provide only themselves; querying the repository
# to figure that out seems unnecessarily complicated
result[i] = (deps.parseDep('trove: %s' % tup[0]),
deps.DependencySet())
needed = [ (i, troveTup) for i, (troveTup, depSets) in
enumerate(izip(troveTupList, result))
if missingNeeded(depSets) ]
if not needed:
return result
# use the getDepsForTroveList call; it raises an error if it needs
# to access some repositories which don't support it
log.info("Getting deps for %d troves" % len(needed))
try:
depList = self.troveSource.getDepsForTroveList(
[ x[1] for x in needed ],
provides = provides,
requires = requires)
except netclient.PartialResultsError, e:
# we can't use this call everywhere; handle what we can and we'll
# deal with the None's later
depList = e.partialResults
def getDepsForTroveList(self, troveTupList, provides = True,
requires = True):
def missingNeeded(depTuple):
if depTuple is None: return True
if provides and depTuple[0] is None: return True
if requires and depTuple[1] is None: return True
return False
def mergeCacheEntry(troveTup, depTuple):
existing = self.depCache.get(depTuple)
if existing is None:
self.depCache[troveTup] = depInfo
else:
self.depCache[troveTup] = (depTuple[0] or existing[0],
depTuple[1] or existing[1])
# look in the dep cache and trove cache
result = [ None ] * len(troveTupList)
for i, tup in enumerate(troveTupList):
result[i] = self.getDepCacheEntry(tup)
if result[i] is None and self.troveIsCached(tup):
trv = self.cache[tup]
result[i] = (trv.getProvides(), trv.getRequires())
elif result[i] is None and trove.troveIsPackage(tup[0]):
# packages provide only themselves; querying the repository
# to figure that out seems unnecessarily complicated
result[i] = (deps.parseDep('trove: %s' % tup[0]),
deps.DependencySet())
needed = [ (i, troveTup) for i, (troveTup, depSets) in
enumerate(izip(troveTupList, result))
if missingNeeded(depSets) ]
if not needed:
return result
# use the getDepsForTroveList call; it raises an error if it needs
# to access some repositories which don't support it
log.info("Getting deps for %d troves" % len(needed))
try:
depList = self.troveSource.getDepsForTroveList(
[ x[1] for x in needed ],
provides = provides,
requires = requires)
except netclient.PartialResultsError, e:
# we can't use this call everywhere; handle what we can and we'll
# deal with the None's later
depList = e.partialResults
def fix(self):
from conary.repository.netrepos import accessmap
db = self.getDB()
ri = accessmap.RoleInstances(db)
for (permissionId, roleId, role) in self._status:
log.info("fixing permission cache for %s...", role)
ri.updatePermissionId(permissionId, roleId)
log.info("checking again to verify changes...")
self._status = set()
if not self.check():
log.error("FAILED to fix the permissions cache. Unhandled error - contact rPath")
db.rollback()
return False
self.commit()
return True
def check(self):
db = self.getDB()
cu = db.cursor()
log.info("checking for extraneous troveinfo records")
cu.execute(
""" select instanceId, count(*)
from Instances join TroveInfo using(instanceId)
where Instances.isPresent = ?
group by instanceId having count(*) > 0 """,
instances.INSTANCE_PRESENT_MISSING)
self._status = cu.fetchall()
if self._status:
log.warning("found %d non-present troves with troveinfo records",
len(self._status))
return False
return True
def updateFromReloaded(self, newCfg, log):
"""Copy updateable options from a newly reloaded config"""
newCfg.sanityCheck()
newCfg.sanityCheckForStart()
for option in self.keys():
if self[option] == newCfg[option]:
continue
if option not in self._reloadable:
if log:
log.warning("Change of option %s requires a restart", option)
continue
self[option] = newCfg[option]
sio = StringIO()
self.displayKey(option, sio)
if log:
log.info("Configuration changed: %s", sio.getvalue().rstrip())
def _closePackages(self, cache, trv, newTroves = None):
packagesAdded = set()
if newTroves is None:
newTroves = list(trv.iterTroveList(strongRefs = True))
for n, v, f in newTroves:
if trove.troveIsComponent(n):
packageN = n.split(':')[0]
if not trv.hasTrove(packageN, v, f):
log.info("adding package %s for component %s",
packageN, (n, v, f))
trv.addTrove(packageN, v, f)
packagesAdded.add( (packageN, v, f) )
cache.cacheComponentMap(packagesAdded)
return packagesAdded
def _closePackages(self, cache, trv, newTroves=None):
packagesAdded = set()
if newTroves is None:
newTroves = list(trv.iterTroveList(strongRefs=True))
for n, v, f in newTroves:
if trove.troveIsComponent(n):
packageN = n.split(':')[0]
if not trv.hasTrove(packageN, v, f):
log.info("adding package %s for component %s", packageN,
(n, v, f))
trv.addTrove(packageN, v, f)
packagesAdded.add((packageN, v, f))
cache.cacheComponentMap(packagesAdded)
return packagesAdded
def fix(self):
from conary.repository.netrepos import accessmap
db = self.getDB()
ri = accessmap.RoleInstances(db)
for (permissionId, roleId, role) in self._status:
log.info("fixing permission cache for %s...", role)
ri.updatePermissionId(permissionId, roleId)
log.info("checking again to verify changes...")
self._status = set()
if not self.check():
log.error(
"FAILED to fix the permissions cache. Unhandled error - contact rPath"
)
db.rollback()
return False
self.commit()
return True
def cmlFindAction(self, actionList, data):
troveset.FindAction.__call__(self, actionList, data)
fetchActions = []
for action in actionList:
action.outSet.realized = True
newAction = troveset.FetchAction(action.outSet, all=True)
newAction.getResultTupleSet(action.primaryTroveSet.g)
fetchActions.append(newAction)
troveset.FetchAction.__call__(fetchActions[0], fetchActions, data)
redirects = []
for action in actionList:
installSet = set()
optionalSet = set()
for troveTup, inInstall in (itertools.chain(
itertools.izip(action.outSet.installSet,
itertools.repeat(True)),
itertools.izip(action.outSet.optionalSet,
itertools.repeat(True)))):
assert (data.troveCache.troveIsCached(troveTup))
trv = data.troveCache.getTrove(withFiles=False, *troveTup)
if trv.isRedirect():
log.info("following redirect %s=%s[%s]", *troveTup)
redirects.append((troveTup, inInstall))
elif inInstall:
installSet.add(troveTup)
else:
optionalSet.add(troveTup)
action.outSet.installSet.clear()
action.outSet.optionalSet.clear()
# caller gets to set this for us
action.realized = False
self._redirects(data, redirects, optionalSet, installSet)
action.outSet._setOptional(optionalSet)
action.outSet._setInstall(installSet)
return True
def cmlFindAction(self, actionList, data):
troveset.FindAction.__call__(self, actionList, data)
fetchActions = []
for action in actionList:
action.outSet.realized = True
newAction = troveset.FetchAction(action.outSet, all = True)
newAction.getResultTupleSet(action.primaryTroveSet.g)
fetchActions.append(newAction)
troveset.FetchAction.__call__(fetchActions[0], fetchActions, data)
redirects = []
for action in actionList:
installSet = set()
optionalSet = set()
for troveTup, inInstall in ( itertools.chain(
itertools.izip( action.outSet.installSet,
itertools.repeat(True)),
itertools.izip( action.outSet.optionalSet,
itertools.repeat(True)) ) ):
assert(data.troveCache.troveIsCached(troveTup))
trv = data.troveCache.getTrove(withFiles = False, *troveTup);
if trv.isRedirect():
log.info("following redirect %s=%s[%s]", *troveTup)
redirects.append( (troveTup, inInstall) )
elif inInstall:
installSet.add(troveTup)
else:
optionalSet.add(troveTup)
action.outSet.installSet.clear()
action.outSet.optionalSet.clear()
# caller gets to set this for us
action.realized = False
self._redirects(data, redirects, optionalSet, installSet)
action.outSet._setOptional(optionalSet)
action.outSet._setInstall(installSet)
return True
def unpackSources(self, resume=None, downloadOnly=False):
if resume == "policy":
return
elif resume:
log.info("Resuming on line(s) %s" % resume)
# note resume lines must be in order
self.processResumeList(resume)
for source in self.iterResumeList(self._sources):
source.doPrep()
source.doAction()
elif downloadOnly:
for source in self._sources:
source.doPrep()
source.doDownload()
else:
for source in self._sources:
source.doPrep()
source.doAction()
def unpackSources(self, resume=None, downloadOnly=False):
if resume == 'policy':
return
elif resume:
log.info("Resuming on line(s) %s" % resume)
# note resume lines must be in order
self.processResumeList(resume)
for source in self.iterResumeList(self._sources):
source.doPrep()
source.doAction()
elif downloadOnly:
for source in self._sources:
source.doPrep()
source.doDownload()
else:
for source in self._sources:
source.doPrep()
source.doAction()
def _caching(self, troveTupList):
local = [ x for x in troveTupList if x[1].isOnLocalHost() ]
if local:
troves = self.db.getTroves(local)
gotTups = []
gotTrvs = []
for troveTup, trv in itertools.izip(local, troves):
if trv is None:
continue
gotTups.append(troveTup)
gotTrvs.append(trv)
troveTupList.remove(troveTup)
self._addToCache(gotTups, gotTrvs)
if troveTupList:
log.info("loading %d trove(s) from the repository, "
"one of which is %s", len(troveTupList), troveTupList[0])
def cacheFilePath(self, cachePrefix, url):
cachePath = self.getCachePath(cachePrefix, url)
util.mkdirChain(os.path.dirname(cachePath))
if url.filePath() in self.cacheMap:
# don't check sha1 twice
return self.cacheMap[url.filePath()]
(troveName, troveVersion, pathId, troveFile, fileId, troveFileVersion,
sha1, mode) = self.nameMap[url.filePath()]
sha1Cached = None
cachedMode = None
if os.path.exists(cachePath):
sha1Cached = sha1helper.sha1FileBin(cachePath)
if sha1Cached != sha1:
if sha1Cached:
log.info('%s sha1 %s != %s; fetching new...', url.filePath(),
sha1helper.sha1ToString(sha1),
sha1helper.sha1ToString(sha1Cached))
else:
log.info('%s not yet cached, fetching...', url.filePath())
if self.quiet:
csCallback = None
else:
csCallback = ChangesetCallback()
f = self.repos.getFileContents([(fileId, troveFileVersion)],
callback=csCallback)[0].get()
outF = util.AtomicFile(cachePath, chmod=0644)
util.copyfileobj(f, outF)
outF.commit()
fileObj = self.repos.getFileVersion(pathId, fileId,
troveFileVersion)
fileObj.chmod(cachePath)
cachedMode = os.stat(cachePath).st_mode & 0777
if mode != cachedMode:
os.chmod(cachePath, mode)
self.cacheMap[url.filePath()] = cachePath
return cachePath
def cacheFilePath(self, cachePrefix, url):
cachePath = self.getCachePath(cachePrefix, url)
util.mkdirChain(os.path.dirname(cachePath))
if url.filePath() in self.cacheMap:
# don't check sha1 twice
return self.cacheMap[url.filePath()]
(troveName, troveVersion, pathId, troveFile, fileId,
troveFileVersion, sha1, mode) = self.nameMap[url.filePath()]
sha1Cached = None
cachedMode = None
if os.path.exists(cachePath):
sha1Cached = sha1helper.sha1FileBin(cachePath)
if sha1Cached != sha1:
if sha1Cached:
log.info('%s sha1 %s != %s; fetching new...', url.filePath(),
sha1helper.sha1ToString(sha1),
sha1helper.sha1ToString(sha1Cached))
else:
log.info('%s not yet cached, fetching...', url.filePath())
if self.quiet:
csCallback = None
else:
csCallback = ChangesetCallback()
f = self.repos.getFileContents(
[(fileId, troveFileVersion)], callback=csCallback)[0].get()
outF = util.AtomicFile(cachePath, chmod=0644)
util.copyfileobj(f, outF)
outF.commit()
fileObj = self.repos.getFileVersion(
pathId, fileId, troveFileVersion)
fileObj.chmod(cachePath)
cachedMode = os.stat(cachePath).st_mode & 0777
if mode != cachedMode:
os.chmod(cachePath, mode)
self.cacheMap[url.filePath()] = cachePath
return cachePath
def _caching(self, troveTupList):
local = [x for x in troveTupList if x[1].isOnLocalHost()]
if local:
troves = self.db.getTroves(local)
gotTups = []
gotTrvs = []
for troveTup, trv in itertools.izip(local, troves):
if trv is None:
continue
gotTups.append(troveTup)
gotTrvs.append(trv)
troveTupList.remove(troveTup)
self._addToCache(gotTups, gotTrvs)
if troveTupList:
log.info(
"loading %d trove(s) from the repository, "
"one of which is %s", len(troveTupList), troveTupList[0])
def _simpleTroveList(self, troveList, newFilesByTrove):
log.info('Verifying %s' % " ".join(x[1].getName() for x in troveList))
changedTroves = set()
try:
result = update.buildLocalChanges(self.db, troveList,
root=self.cfg.root,
forceSha1=self.forceHashCheck,
ignoreTransient=True,
updateContainers=True,
statCache = self.statCache)
if not result: return
cs = result[0]
changed = False
for (changed, trv) in result[1]:
if changed:
changedTroves.add(trv.getNameVersionFlavor())
except OSError, err:
if err.errno == 13:
log.warning("Permission denied creating local changeset for"
" %s " % str([ x[0].getName() for x in troveList ]))
return
def updateRecipes(repos, cfg, recipeList, committedSources):
committedSourcesByNB = {}
for name, version, flavor in committedSources:
committedSourcesByNB[name, version.branch()] = version
for recipe in recipeList:
recipeDir = os.path.dirname(recipe)
stateFilePath = recipeDir + '/CONARY'
if not os.path.exists(stateFilePath):
continue
conaryStateFile = state.ConaryStateFromFile(stateFilePath)
if not conaryStateFile.hasSourceState():
continue
context = conaryStateFile.getContext()
stateFile = conaryStateFile.getSourceState()
troveName = stateFile.getName()
branch = stateFile.getBranch()
if (troveName, branch) not in committedSourcesByNB:
continue
stateVersion = stateFile.getVersion()
newVersion = committedSourcesByNB[troveName, branch]
if stateVersion != versions.NewVersion():
log.info('Updating %s after commit' % recipeDir)
if compat.ConaryVersion().updateSrcTakesMultipleVersions():
try:
# Added in CNY-3035
checkin.nologUpdateSrc(repos, [recipeDir])
except checkin.builderrors.UpToDate:
pass # Don't mention if the source is already up to date
except checkin.builderrors.CheckinError, e:
e.logError()
except AttributeError:
checkin.updateSrc(repos, [recipeDir])
else:
curDir = os.getcwd()
try:
os.chdir(recipeDir)
checkin.updateSrc(repos)
finally:
os.chdir(curDir)
def check(self):
db = self.getDB()
cu = db.cursor()
log.info("checking existing Permissions cache")
cu.execute(
"""
select p.permissionId, p.userGroupId, ug.userGroup, i.item, l.label, coalesce(ugap.c,0)
from Permissions as p
join UserGroups as ug using (userGroupId)
join Items as i on p.itemId = i.itemId
join Labels as l on p.labelId = l.labelId
left join (
select permissionId, count(*) as c
from UserGroupAllPermissions
join Instances using(instanceId)
where Instances.isPresent != ?
group by permissionId ) as ugap on p.permissionId = ugap.permissionId
""", instances.INSTANCE_PRESENT_MISSING)
info = {}
existing = {}
for permissionId, roleId, role, item, label, count in cu:
info[permissionId] = (roleId, role, item, label)
existing[permissionId] = count
log.info("checking for missing Permissions caches...")
cu.execute(
"""
select p.permissionId, coalesce(checker.c,0)
from Permissions as p
left join (
select permissionId, count(*) as c from (
select Permissions.permissionId as permissionId,
Instances.instanceId as instanceId
from Instances
join Nodes using(itemId, versionId)
join LabelMap using(itemId, branchId)
join Permissions on
Permissions.labelId = 0 or Permissions.labelId = LabelMap.labelId
join CheckTroveCache on
Permissions.itemId = CheckTroveCache.patternId and
Instances.itemId = CheckTroveCache.itemId
where Instances.isPresent != ?
) as perms
group by permissionId ) as checker using (permissionId)
""", instances.INSTANCE_PRESENT_MISSING)
self._status = set()
ret = True
for permissionId, newCounter in cu:
crtCounter = existing.get(permissionId, 0)
if crtCounter == newCounter:
continue
roleId, role, item, label = info[permissionId]
log.warning(
"acl(%d) (%s %s %s) caches %d entries instead of %d entries",
permissionId, role, label, item, crtCounter, newCounter)
self._status.add((permissionId, roleId, role))
ret = False
if not ret:
log.info("check fails with %d errors found", len(self._status))
return ret
def check(self):
db = self.getDB()
log.info("checking the state of the CheckTroveCache table")
cu = db.cursor()
cu.execute("select patternId, itemId from CheckTroveCache")
existing = set([(x[0],x[1]) for x in cu.fetchall()])
required = []
cu.execute("select distinct i.itemId, i.item from Permissions as p "
"join Items as i using(itemId)")
patterns = set([(x[0], x[1]) for x in cu.fetchall()])
cu.execute("select itemId, item from Items")
troveNames = set([(x[0], x[1]) for x in cu.fetchall()])
for patternId, pattern in patterns:
for itemId, item in troveNames:
if items.checkTrove(pattern, item):
required.append((patternId, itemId))
required = set(required)
self._status = required.difference(existing)
if len(self._status):
log.warning("found %d entries that are missing from CheckTroveCache", len(self._status))
return False
return True
def _simpleTroveList(self, troveList, newFilesByTrove):
log.info('Verifying %s' % " ".join(x[1].getName() for x in troveList))
changedTroves = set()
try:
result = update.buildLocalChanges(self.db,
troveList,
root=self.cfg.root,
forceSha1=self.forceHashCheck,
ignoreTransient=True,
updateContainers=True,
statCache=self.statCache)
if not result: return
cs = result[0]
changed = False
for (changed, trv) in result[1]:
if changed:
changedTroves.add(trv.getNameVersionFlavor())
except OSError, err:
if err.errno == 13:
log.warning("Permission denied creating local changeset for"
" %s " % str([x[0].getName() for x in troveList]))
return
def findAvailableTargetFlavors(self, repos):
if self.branchStr is None:
# redirect to nothing
return set()
if self.branchStr[0] == "/":
branch = versions.VersionFromString(self.branchStr)
if not isinstance(branch, versions.Branch):
raise builderrors.RecipeFileError, "Redirects must specify branches or labels, " "not versions"
log.info("redirecting to branches is deprecated; redirects must " "be to labels")
matches = repos.getTroveLeavesByBranch({self.destName: {branch: None}})
else:
label = versions.Label(self.branchStr)
matches = repos.getTroveLatestByLabel({self.destName: {label: None}})
targetFlavors = set()
# Get the flavors and branch available on the target
for version, flavorList in matches.get(self.destName, {}).iteritems():
targetFlavors.update((version, x) for x in flavorList)
return targetFlavors
def check(self):
db = self.getDB()
cu = db.cursor()
# determine what entries (if any) are visible from LatestView
# but aren't cached into LatestCache
log.info("checking if the LatestCache table is current...")
cu.execute("""
select userGroupId, itemId, branchId, flavorId, versionId, latestType, count(*) as c
from ( select userGroupId, itemId, branchId, flavorId, versionId, latestType from latestview
union all
select userGroupId, itemId, branchId, flavorId, versionId, latestType from latestcache
) as duplicates
group by userGroupId, itemId, branchId, flavorId, versionId, latestType
having count(*) != 2 """)
# any entry that does not appear twice is cached wrong
self._status = set()
for userGroupId, itemId, branchId, flavorId, versionId, latestType, c in cu:
# record what needs rebuilding
self._status.add((itemId, branchId, flavorId))
if self._status:
log.info("detected %d LatestCache entries that need correction" %
(len(self._status), ))
return False
return True
def check(self):
db = self.getDB()
cu = db.cursor()
log.info("checking existing Permissions cache")
cu.execute("""
select p.permissionId, p.userGroupId, ug.userGroup, i.item, l.label, coalesce(ugap.c,0)
from Permissions as p
join UserGroups as ug using (userGroupId)
join Items as i on p.itemId = i.itemId
join Labels as l on p.labelId = l.labelId
left join (
select permissionId, count(*) as c
from UserGroupAllPermissions
join Instances using(instanceId)
where Instances.isPresent != ?
group by permissionId ) as ugap on p.permissionId = ugap.permissionId
""", instances.INSTANCE_PRESENT_MISSING)
info = {}
existing = {}
for permissionId, roleId, role, item, label, count in cu:
info[permissionId] = (roleId, role, item, label)
existing[permissionId] = count
log.info("checking for missing Permissions caches...")
cu.execute("""
select p.permissionId, coalesce(checker.c,0)
from Permissions as p
left join (
select permissionId, count(*) as c from (
select Permissions.permissionId as permissionId,
Instances.instanceId as instanceId
from Instances
join Nodes using(itemId, versionId)
join LabelMap using(itemId, branchId)
join Permissions on
Permissions.labelId = 0 or Permissions.labelId = LabelMap.labelId
join CheckTroveCache on
Permissions.itemId = CheckTroveCache.patternId and
Instances.itemId = CheckTroveCache.itemId
where Instances.isPresent != ?
) as perms
group by permissionId ) as checker using (permissionId)
""", instances.INSTANCE_PRESENT_MISSING)
self._status = set()
ret = True
for permissionId, newCounter in cu:
crtCounter = existing.get(permissionId, 0)
if crtCounter == newCounter:
continue
roleId, role, item, label = info[permissionId]
log.warning("acl(%d) (%s %s %s) caches %d entries instead of %d entries",
permissionId, role, label, item, crtCounter, newCounter)
self._status.add((permissionId, roleId, role))
ret = False
if not ret:
log.info("check fails with %d errors found", len(self._status))
return ret
def check(self):
db = self.getDB()
cu = db.cursor()
# determine what entries (if any) are visible from LatestView
# but aren't cached into LatestCache
log.info("checking if the LatestCache table is current...")
cu.execute("""
select userGroupId, itemId, branchId, flavorId, versionId, latestType, count(*) as c
from ( select userGroupId, itemId, branchId, flavorId, versionId, latestType from latestview
union all
select userGroupId, itemId, branchId, flavorId, versionId, latestType from latestcache
) as duplicates
group by userGroupId, itemId, branchId, flavorId, versionId, latestType
having count(*) != 2 """)
# any entry that does not appear twice is cached wrong
self._status = set()
for userGroupId, itemId, branchId, flavorId, versionId, latestType, c in cu:
# record what needs rebuilding
self._status.add((itemId, branchId, flavorId))
if self._status:
log.info("detected %d LatestCache entries that need correction" % (
len(self._status),))
return False
return True
def check(self):
db = self.getDB()
log.info("checking the state of the CheckTroveCache table")
cu = db.cursor()
cu.execute("select patternId, itemId from CheckTroveCache")
existing = set([(x[0], x[1]) for x in cu.fetchall()])
required = []
cu.execute("select distinct i.itemId, i.item from Permissions as p "
"join Items as i using(itemId)")
patterns = set([(x[0], x[1]) for x in cu.fetchall()])
cu.execute("select itemId, item from Items")
troveNames = set([(x[0], x[1]) for x in cu.fetchall()])
for patternId, pattern in patterns:
for itemId, item in troveNames:
if items.checkTrove(pattern, item):
required.append((patternId, itemId))
required = set(required)
self._status = required.difference(existing)
if len(self._status):
log.warning(
"found %d entries that are missing from CheckTroveCache",
len(self._status))
return False
return True
def _updateTroves(cfg, applyList, **kwargs):
# Take out the apply-related keyword arguments
applyDefaults = dict(
replaceFiles = False,
replaceManagedFiles = False,
replaceUnmanagedFiles = False,
replaceModifiedFiles = False,
replaceModifiedConfigFiles = False,
tagScript = None,
justDatabase = False,
skipCapsuleOps = False,
info = False,
keepJournal = False,
noRestart = False,
noScripts = False,
)
applyKwargs = {}
for k in applyDefaults:
if k in kwargs:
applyKwargs[k] = kwargs.pop(k)
callback = kwargs.pop('callback')
loadTroveCache = kwargs.pop('loadTroveCache', False)
applyKwargs['test'] = kwargs.get('test', False)
applyKwargs['localRollbacks'] = cfg.localRollbacks
applyKwargs['autoPinList'] = cfg.pinTroves
model = kwargs.pop('systemModel', None)
modelFile = kwargs.pop('systemModelFile', None)
modelGraph = kwargs.pop('modelGraph', None)
modelTrace = kwargs.pop('modelTrace', None)
noRestart = applyKwargs.get('noRestart', False)
client = conaryclient.ConaryClient(cfg, modelFile=modelFile)
client.setUpdateCallback(callback)
if kwargs.pop('disconnected', False):
client.disconnectRepos()
migrate = kwargs.get('migrate', False)
# even though we no longer differentiate forceMigrate, we still
# remove it from kwargs to avoid confusing prepareUpdateJob
kwargs.pop('forceMigrate', False)
restartInfo = kwargs.get('restartInfo', None)
# Initialize the critical update set
applyCriticalOnly = kwargs.get('applyCriticalOnly', False)
if kwargs.get('criticalUpdateInfo') is not None:
kwargs['criticalUpdateInfo'].criticalOnly = applyCriticalOnly
else:
kwargs['criticalUpdateInfo'] = CriticalUpdateInfo(applyCriticalOnly)
info = applyKwargs.pop('info', False)
# Rename depCheck to resolveDeps
depCheck = kwargs.pop('depCheck', True)
kwargs['resolveDeps'] = depCheck
if not info:
client.checkWriteableRoot()
# Unfortunately there's no easy way to make 'test' or 'info' mode work
# with capsule sync, doubly so because it influences the decisions made
# later on about what troves to update. So this will always really
# apply, but the good news is that it never modifies the system outside
# of the Conary DB.
client.syncCapsuleDatabase(callback, makePins=True)
updJob = client.newUpdateJob()
try:
if model:
changeSetList = kwargs.get('fromChangesets', [])
criticalUpdates = kwargs.get('criticalUpdateInfo', None)
tc = modelupdate.CMLTroveCache(client.getDatabase(),
client.getRepos(),
callback = callback,
changeSetList =
changeSetList)
tcPath = cfg.root + cfg.dbPath + '/modelcache'
if loadTroveCache:
if os.path.exists(tcPath):
log.info("loading %s", tcPath)
callback.loadingModelCache()
tc.load(tcPath)
ts = client.cmlGraph(model, changeSetList = changeSetList)
if modelGraph is not None:
ts.g.generateDotFile(modelGraph)
suggMap = client._updateFromTroveSetGraph(updJob, ts, tc,
fromChangesets = changeSetList,
criticalUpdateInfo = criticalUpdates,
callback = callback)
if modelTrace is not None:
ts.g.trace([ parseTroveSpec(x) for x in modelTrace ] )
finalModel = copy.deepcopy(model)
if model.suggestSimplifications(tc, ts.g):
log.info("possible system model simplifications found")
ts2 = client.cmlGraph(model, changeSetList = changeSetList)
updJob2 = client.newUpdateJob()
try:
suggMap2 = client._updateFromTroveSetGraph(updJob2, ts2,
tc,
fromChangesets = changeSetList,
criticalUpdateInfo = criticalUpdates)
except errors.TroveNotFound:
log.info("bad model generated; bailing")
else:
if (suggMap == suggMap2 and
updJob.getJobs() == updJob2.getJobs()):
log.info("simplified model verfied; using it instead")
ts = ts2
finalModel = model
updJob = updJob2
suggMap = suggMap2
else:
log.info("simplified model changed result; ignoring")
model = finalModel
modelFile.model = finalModel
if tc.cacheModified():
log.info("saving %s", tcPath)
callback.savingModelCache()
tc.save(tcPath)
callback.done()
else:
suggMap = client.prepareUpdateJob(updJob, applyList, **kwargs)
except:
callback.done()
client.close()
raise
if info:
callback.done()
displayUpdateInfo(updJob, cfg, noRestart=noRestart)
if restartInfo and not model:
callback.done()
newJobs = set(itertools.chain(*updJob.getJobs()))
oldJobs = set(updJob.getItemList())
addedJobs = newJobs - oldJobs
removedJobs = oldJobs - newJobs
if addedJobs or removedJobs:
print
print 'NOTE: after critical updates were applied, the contents of the update were recalculated:'
print
displayChangedJobs(addedJobs, removedJobs, cfg)
updJob.close()
client.close()
return
if model:
missingLocalTroves = model.getMissingLocalTroves(tc, ts)
if missingLocalTroves:
print 'Update would leave references to missing local troves:'
for troveTup in missingLocalTroves:
if not isinstance(troveTup, trovetup.TroveTuple):
troveTup = trovetup.TroveTuple(troveTup)
print "\t" + str(troveTup)
client.close()
return
if suggMap:
callback.done()
dcfg = display.DisplayConfig()
dcfg.setTroveDisplay(fullFlavors = cfg.fullFlavors,
fullVersions = cfg.fullVersions,
showLabels = cfg.showLabels)
formatter = display.TroveTupFormatter(dcfg)
print "Including extra troves to resolve dependencies:"
print " ",
items = sorted(set(formatter.formatNVF(*x)
for x in itertools.chain(*suggMap.itervalues())))
print " ".join(items)
askInteractive = cfg.interactive
if restartInfo:
callback.done()
newJobs = set(itertools.chain(*updJob.getJobs()))
oldJobs = set(updJob.getItemList())
addedJobs = newJobs - oldJobs
removedJobs = oldJobs - newJobs
if not model and addedJobs or removedJobs:
print 'NOTE: after critical updates were applied, the contents of the update were recalculated:'
displayChangedJobs(addedJobs, removedJobs, cfg)
else:
askInteractive = False
if not updJob.jobs:
# Nothing to do
print 'Update would not modify system'
if model and not kwargs.get('test'):
# Make sure 'conary sync' clears model.next even if nothing needs
# to be done.
modelFile.closeSnapshot()
updJob.close()
client.close()
return
elif askInteractive:
print 'The following updates will be performed:'
displayUpdateInfo(updJob, cfg, noRestart=noRestart)
if migrate and cfg.interactive:
print ('Migrate erases all troves not referenced in the groups'
' specified.')
if askInteractive:
if migrate:
style = 'migrate'
else:
style = 'update'
okay = cmdline.askYn('continue with %s? [Y/n]' % style, default=True)
if not okay:
updJob.close()
client.close()
return
if not noRestart and updJob.getCriticalJobs():
print "Performing critical system updates, will then restart update."
try:
restartDir = client.applyUpdateJob(updJob, **applyKwargs)
finally:
updJob.close()
client.close()
if restartDir:
params = sys.argv
# Write command line to disk
import xmlrpclib
cmdlinefile = open(os.path.join(restartDir, 'cmdline'), "w")
cmdlinefile.write(xmlrpclib.dumps((params, ), methodresponse = True))
cmdlinefile.close()
# CNY-980: we should have the whole script of changes to perform in
# the restart directory (in the job list); if in migrate mode, re-exec
# as regular update
if migrate and 'migrate' in params:
params[params.index('migrate')] = 'update'
params.extend(['--restart-info=%s' % restartDir])
client.close()
raise errors.ReexecRequired(
'Critical update completed, rerunning command...', params,
restartDir)
else:
if (not kwargs.get('test', False)) and model:
modelFile.closeSnapshot()
