def __init__(self, logger):
self._log = logger.createLogger("sys-install", "sub-pkg-mng")
self.utils = Utils(self._log)
self.useFakeChrootMode = False
self.rootDir = None
self.subPkgDir = None
self.subPkgTempDir = None
def __init__ (self, logger, tempDir):
self._log=logger.createLogger("sys-install","yum-yum")
self.utils=Utils(self._log)
self.rootDir="/"
self.useFakeChrootMode=False
self.diskRepos=[]
self.httpRepos=[]
self.tempDir=tempDir
# tempDir MUST begin with '/', see _getConfPath
self._raiseIfNotAbsPath(tempDir, "tempDir")
self.yumConfPath=os.path.join(self.tempDir, "yum.conf")
self.yumConfPluginDir=os.path.join(self.tempDir, "yum.pluginconf.d")
self._updateCommand()
def __init__ (self, logger):
self._log=logger.createLogger("sys-install","sub-pkg-mng")
self.utils=Utils(self._log)
self.useFakeChrootMode=False
self.rootDir=None
self.subPkgDir=None
self.subPkgTempDir=None
def __init__(self, logger, stateManager, installManager, pilotVersion,
pilotDir, logDir, workDir, apiDir, startupScriptsDir,
rpmActiveRfs, yumActiveRfs, yumConfForActiveRfs,
yumConfForNewVersionOnly, prevRfsDir, sdUtils, bootUtils):
self._log = logger.createLogger("sys-install", "pilot-services")
self._utils = Utils(self._log)
self._stateManager = stateManager
self._installManager = installManager
self._pilotVersion = pilotVersion
self._pilotDir = pilotDir
self._logDir = logDir
self._workDir = workDir
self._apiDir = apiDir
self._startupScriptsDir = startupScriptsDir
self._rpmActiveRfs = rpmActiveRfs
self._yumActiveRfs = yumActiveRfs
self._yumConfForActiveRfs = yumConfForActiveRfs
self._yumConfForNewVersionOnly = yumConfForNewVersionOnly
self._debugNoInstallMode = False
self._otherPilot = None
self._pilotHints = None
self._prevRfsDir = prevRfsDir
self._sdUtils = sdUtils
self._bootUtils = bootUtils
def __init__ (self, logger, stateManager, installManager, pilotVersion, pilotDir, logDir, workDir, apiDir, startupScriptsDir,
rpmActiveRfs, yumActiveRfs, yumConfForActiveRfs, yumConfForNewVersionOnly, prevRfsDir, sdUtils, bootUtils):
self._log = logger.createLogger("sys-install","pilot-services")
self._utils = Utils(self._log)
self._stateManager = stateManager
self._installManager = installManager
self._pilotVersion = pilotVersion
self._pilotDir = pilotDir
self._logDir = logDir
self._workDir = workDir
self._apiDir = apiDir
self._startupScriptsDir = startupScriptsDir
self._rpmActiveRfs = rpmActiveRfs
self._yumActiveRfs = yumActiveRfs
self._yumConfForActiveRfs = yumConfForActiveRfs
self._yumConfForNewVersionOnly = yumConfForNewVersionOnly
self._debugNoInstallMode = False
self._otherPilot = None
self._pilotHints = None
self._prevRfsDir = prevRfsDir
self._sdUtils = sdUtils
self._bootUtils = bootUtils
class Yum(object):
configTemplate="""
[main]
cachedir=/var/cache/yum
# We do not want to waste space: cache should be clear after installation
keepcache=0
debuglevel=2
logfile=/var/log/yum.log
exactarch=1
obsoletes=1
plugins=1
pluginpath=%(pluginpath)s
pluginconfpath=%(pluginpath)s
#rpmverbosity=debug
# List of 'provides' of packages that are allowed to be installed multiple versions side by side
installonlypkgs=qb-allow-side-by-side
# install-only packages are allowed until the sun burns down
installonly_limit=999
gpgcheck=1
localpkg_gpgcheck=1
"""
configRepoTemplate="""
[%(repoName)s]
name=%(repoName)s
baseurl=%(baseurl)s
"""
def __init__ (self, logger, tempDir):
self._log=logger.createLogger("sys-install","yum-yum")
self.utils=Utils(self._log)
self.rootDir="/"
self.useFakeChrootMode=False
self.diskRepos=[]
self.httpRepos=[]
self.tempDir=tempDir
# tempDir MUST begin with '/', see _getConfPath
self._raiseIfNotAbsPath(tempDir, "tempDir")
self.yumConfPath=os.path.join(self.tempDir, "yum.conf")
self.yumConfPluginDir=os.path.join(self.tempDir, "yum.pluginconf.d")
self._updateCommand()
def setRoot (self, rootDir, useFakeChrootMode=False):
self._raiseIfNotAbsPath(rootDir, "rootDir")
self.rootDir=rootDir
self.useFakeChrootMode=useFakeChrootMode
self._updateCommand()
def addDiskRepo (self, repoName, repoDir):
self._raiseIfNotAbsPath(repoDir, "repoDir")
self.diskRepos.append((repoName, repoDir))
if self.useFakeChrootMode:
self._testModeAddRpms(repoName, repoDir)
def addHttpRepo (self, repoName, repoAddr):
self.httpRepos.append((repoName, repoAddr))
def doInstall (self, package):
self._log("do-install-called").info("doInstall(%s) called", package)
self._createYumConf()
cmd="--assumeyes install "+package
#Work around to the ANNOYING problem of yum refusing to install a leader because it requires an oscar which is older
# than some installed oscar. Instead of raising if install fails, we check the output, and if it failed due to
# some 'newer' package which is already installed, we install that package first, and retry
#self._runYumRaiseIfFail("--assumeyes install "+package)
retry=True
numLoops=0
while retry:
retry=False
(rc,outText,errText)=self._runYum(cmd)
if rc==0:
return
# This is a sample of the error message we get from yum:
#package oscar-2.1.0.1-12386.x86_64 (which is newer than oscar-2.1.0.0-20120320133848.x86_64) is already installed
mo=re.search("which is newer than (.*)\) is already installed", errText)
if mo != None:
rpmToInstall=mo.group(1)
self._log("do-install-workaround").info("doInstall(%s) workaround: Found package %s that needs to be installed first", package, rpmToInstall)
# We assume that this package must be installed properly, without tricks
self._runYumRaiseIfFail("--assumeyes install "+rpmToInstall)
numLoops += 1
if numLoops < 10:
retry=True
else:
self._log("do-install-workaround").info("doInstall(%s) workaround: Did not find any package that needs to be installed first", package)
raise RuntimeError("Yum install failed: returned %s" % rc)
def doReinstall (self, package):
self._log("do-reinstall-called").info("doReinstall(%s) called", package)
self._createYumConf()
self._runYumRaiseIfFail("--assumeyes reinstall "+package)
def doUpdate (self, package):
self._log("do-update-called").info("doUpdate(%s) called", package)
self._createYumConf()
self._runYumRaiseIfFail("--assumeyes update "+package)
def doRemove (self, package):
self._log("do-remove-called").info("doRemove(%s) called", package)
self._createYumConf()
self._runYumRaiseIfFail("--assumeyes remove "+package)
def doUpgrade (self, package):
self._log("do-upgrade-called").info("doUpgrade(%s) called", package)
self._createYumConf()
self._runYumRaiseIfFail("--assumeyes upgrade "+package)
# localinstall is needed because 'yum install' with a file name works only if file anme ends with .rpm
def doLocalInstall (self, package):
self._log("do-local-install-called").info("doLocalInstall(%s) called", package)
self._createYumConf()
self._runYumRaiseIfFail("--assumeyes localinstall "+package)
def doTestInstall (self, package):
"""Checks that an install would work. Returns True if OK, False if not"""
self._log("do-local-install-called").info("doTestInstall(%s) called", package)
self._createYumConf()
(rc,outText,errText)=self._runYum("--assumeyes --downloadonly install "+package)
# 0 means nothing to do. Message in output means exit due to plugin, not due to errors
ret = ((rc==0) or (errText.find("exiting because --downloadonly specified") != -1))
self._log("do-local-install-done").info("doTestInstall() returning %s", ret)
return ret
def createYumConfAt (self, yumConfDir):
"""
Given a directory name, creates a yum.conf file in it (Plus plugins, as needed).
Returns name of generated yum.conf file (To be passed to 'yum -c')
"""
self._log("create-yum-conf-at-called").info("createYumConfAt() called, dir=%s", yumConfDir)
pathToYumConf=os.path.join(yumConfDir, "yum.conf")
if not os.path.exists(yumConfDir):
os.makedirs(yumConfDir)
pathToYumPluginConf = os.path.join(yumConfDir, "yum.pluginconf.d")
if not os.path.exists(pathToYumPluginConf):
os.makedirs(pathToYumPluginConf)
confText=self._getConfText() % {'pluginpath' : pathToYumPluginConf}
self._writeTextToFile(confText, pathToYumConf)
self._log("create-yum-conf-at-text").info("createYumConfAt() called, conf text is '%s'", confText)
self._createYumPlugins(pathToYumPluginConf)
self._log("create-yum-conf-at-done").info("createYumConfAt() done")
return pathToYumConf
def _testModeAddRpms (self, repoName, repoDir):
"""For unit tests only, copies rpms to yum cache to avoid bug"""
# This is needed to overcome a bug in fakechroot (apparently): If the rpm itself is not found in the cache,
# Then yum checks for the space available on disk before 'downloading it', and it uses the statvfs() for that,
# and apparently this function is not covered by fakechroot. See this strace output:
#
# 10887 stat("/users/eng/orens/testRpm/rfs1/var/cache/yum/x86_64/centos6/box/packages/sugar-2.0-2.x86_64.rpm", 0x7fffa3a72260) = -1 ENOENT (No such file or directory)
# 10887 statfs("/var/cache/yum/x86_64/centos6/box/packages", 0x7fffa3a722d0) = -1 ENOENT (No such file or directory)
#
# This is the result of code at /usr/lib/python2.6/site-packages/yum/__init__.py, line 1600.
self._log("test-mode-add-rpms").info("_testModeAddRpms(repoName=%s, repoDir=%s) called, self.rootDir=%s", repoName, repoDir, self.rootDir)
self._raiseIfNotAbsPath(repoDir, "repoDir")
if not self.useFakeChrootMode:
raise ValueError("Never call this func in production code")
repoCachePackagesDirFromOutside=os.path.join(self.rootDir, "var/cache/yum/"+repoName+"/packages")
self.utils.runCommandRaiseIfFail("mkdir -p "+repoCachePackagesDirFromOutside)
if repoDir.startswith(self.rootDir):
repoDir = repoDir.replace(self.rootDir, "")
repoDirFromOutside=self.rootDir+repoDir
for file_ in os.listdir(repoDirFromOutside):
if file_.endswith(".rpm"):
fileFromOutside=os.path.join(repoDirFromOutside, file_)
self.utils.runCommandRaiseIfFail("cp "+fileFromOutside+" "+repoCachePackagesDirFromOutside)
def _getConfPath (self):
pathToYumConf=os.path.join(self.rootDir, self.yumConfPath[1:]) # Remove leading '/' from
return pathToYumConf
def _getPluginConfPath (self):
pathToYumPluginConf=os.path.join(self.rootDir, self.yumConfPluginDir[1:]) # Remove leading '/' from
return pathToYumPluginConf
def _getConfText (self):
text=self.configTemplate
for (repoName, repoDir) in self.diskRepos:
text += self.configRepoTemplate % {'repoName' : repoName, 'baseurl' : 'file://'+repoDir}
for (repoName, repoAddr) in self.httpRepos:
text += self.configRepoTemplate % {'repoName' : repoName, 'baseurl' : 'http://'+repoAddr}
return text
def _createYumPlugins (self, pluginConfDir):
downloadonlyConfText="""
[main]
enabled=1
"""
downloadonlyPyText="""
from yum.plugins import PluginYumExit, TYPE_INTERACTIVE
requires_api_version = '2.1'
plugin_type = (TYPE_INTERACTIVE,)
def config_hook(conduit):
parser = conduit.getOptParser()
if hasattr(parser, 'plugin_option_group'):
parser = parser.plugin_option_group
parser.add_option('', '--downloadonly', dest='dlonly', action='store_true',
default=False, help="don't update, just download")
parser.add_option('', '--downloaddir', dest='dldir',
action='store', default=None,
help="specifies an alternate directory to store packages")
def postreposetup_hook(conduit):
opts, commands = conduit.getCmdLine()
if opts.dldir:
repos = conduit.getRepos()
rlist = repos.listEnabled()
for repo in rlist:
repo.setAttribute('pkgdir',opts.dldir)
def postdownload_hook(conduit):
opts, commands = conduit.getCmdLine()
# Don't die on errors, or we'll never see them.
if not conduit.getErrors() and opts.dlonly:
raise PluginYumExit('exiting because --downloadonly specified ')
"""
self._writeTextToFile(downloadonlyConfText, os.path.join(pluginConfDir, "downloadonly.conf"))
self._writeTextToFile(downloadonlyPyText, os.path.join(pluginConfDir, "downloadonly.py"))
def _createYumConf (self):
pathToYumConf=self._getConfPath()
self._log("create-yum-conf").info("_createYumConf() called, writing to %s", pathToYumConf)
dir_=os.path.dirname(pathToYumConf)
if not os.path.exists(dir_):
os.makedirs(dir_)
pathToYumPluginConf = self._getPluginConfPath()
if not os.path.exists(pathToYumPluginConf):
os.makedirs(pathToYumPluginConf)
confText=self._getConfText() % {'pluginpath' : self.yumConfPluginDir}
self._writeTextToFile(confText, pathToYumConf)
self._log("create-yum-conf-text").info("_createYumConf() called, conf text is '%s'", confText)
self._createYumPlugins(pathToYumPluginConf)
self._log("create-yum-conf-done").info("_createYumConf() done")
def _writeTextToFile (self, text, fileName):
f=open(fileName, "w")
f.write(text)
f.close()
def _removeYumConf (self):
os.remove(self._getConfPath())
self.utils.runCommandRaiseIfFail("rm -rf "+self._getPluginConfPath())
def _raiseIfNotAbsPath (self, pathToCheck, name):
if not pathToCheck.startswith('/'):
raise ValueError("%s must start with '/', value given is '%s'" % (name, pathToCheck))
def _runYum (self, cmd):
"""
NOTE: This function is used by pilot services to allow pilot to run the yum command
"""
cmd=self.command+" "+cmd
(rc,outText,errText)=self.utils.runCommand(cmd)
return (rc,outText,errText)
def _runYumRaiseIfFail (self, cmd):
(rc,outText,errText)=self._runYum(cmd)
if rc != 0:
raise RuntimeError("Yum command '%s' failed: returned %s" % (cmd, rc))
return (outText,errText)
def _updateCommand (self):
self.command="yum -c "+self.yumConfPath
if self.useFakeChrootMode:
self.command="fakeroot fakechroot /usr/sbin/chroot "+self.rootDir+" "+self.command
elif self.rootDir != "/":
self.command = self.command+" --installroot "+self.rootDir
def __init__ (self, logger, product, sysVarDir, sysRunDir, dataVarDir, rpmsDir,
appDir, appDirReal, bootDevPath, grubConf, prevRfsDir=None, vitalDir=None, systemRoot="/", useFakeChrootMode=False):
"""
product : Name of product we handle, e.g. 'qb'.
sysVarDir : path to sys dir relative to root, e.g. '/opt/qb/sys/oscar/install/0/var'.
sysRunDir : path to sys dir relative to root, e.g. '/opt/qb/sys/oscar/install/0/run'.
dataVarDir : path to data dir relative to root, e.g. '/opt/qb/data/oscar/install/0/var'.
rpmsDir : path to rpms dir relative to root, e.g. '/opt/qb/rpms'.
appDir : path to app dir relative to root, e.g. '/opt/qb/app'.
appDirReal : real path to app dir relative to root, e.g. '/opt/qb/sys/rfs/sys/0/var/app'.
bootDevPath : Path to boot device, e.g. '/boot'
grubConf : name of grub.conf file relative to boot device, e.g. 'grub/grub.conf'.
systemRoot : For unit tests, specifies location of the file-system root.
useFakeChrootMode : For unit tests, True to use fakechroot instead of chroot (via rpm --root or yum --installroot)
prevRfsDir : Location to copy some RFS data to during OS install
vitalDir : Path to vital dir.
"""
self._raiseIfNotAbsPath(systemRoot, "systemRoot")
self._log = logger.createLogger("sys-install", "install-manager")
self._log.setInstance(product)
self._utils = Utils(self._log)
self._product = product
self._bootDevPath = bootDevPath
self._grubConf = grubConf
self._systemRoot = systemRoot
self._useFakeChrootMode = useFakeChrootMode
self._vitalDir = vitalDir
self._rpmsDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(rpmsDir))
self._sysVarDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(sysVarDir))
self._sysRunDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(sysRunDir))
self._dataVarDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(dataVarDir))
self._logDir=os.path.join(self._dataVarDir, 'log')
self._stateDir = os.path.join(self._sysVarDir, "state")
self._yumConfActiveRfsDir = os.path.join(self._sysVarDir, "yum-conf-active-rfs")
self._yumConfNewVersionDir = os.path.join(self._sysVarDir, "yum-conf-new-version")
self._appDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(appDir))
self._appDirReal = os.path.join(systemRoot, self._chopLeadingSlashIfFound(appDirReal))
self._installDir = os.path.join(self._sysVarDir,"sub-pkgs")
self._workDir=os.path.join(self._sysVarDir, 'work')
self._apiDir=os.path.join(self._sysVarDir, 'api')
self._startupScriptsDir=os.path.join(self._sysVarDir, 'startup')
self._startupScriptsPrevDir=os.path.join(self._startupScriptsDir, 'prev')
self._rpmKeysDir = os.path.join(self._sysVarDir, "rpm/keys")
self._tempDir = os.path.join(self._sysRunDir, "tmp")
self._installTempDir = os.path.join(self._tempDir, "sub-pkgs") # install-temp can be cleared between runs
self._subPkgManager = SubPkgManager(self._log)
self._subPkgManager.setDirs(self._systemRoot, self._installDir, self._installTempDir, self._rpmKeysDir, self._useFakeChrootMode)
self._stateFile = os.path.join(self._stateDir, product+"-state")
self._stateManager = StateManager(self._log)
self._stateManager.setFile(self._stateFile)
self._pilotHints=""
self._prevRfsDir = prevRfsDir
self._preparedPackageFilePath = None
self._log("init").info("InstallManager::__init__() called, self._product=%s, self._sysVarDir=%s, self._sysRunDir=%s, "\
"self._dataVarDir=%s, self._rpmsDir=%s, self._appDir=%s, self._bootDevPath=%s, "\
"self._grubConf=%s, self._prevRfsDir=%s, self._vitalDir=%s",
self._product, self._sysVarDir, self._sysRunDir, self._dataVarDir, self._rpmsDir,
self._appDir, self._bootDevPath, self._grubConf, self._prevRfsDir, self._vitalDir)
# If true, we will not actually install RPMs. This helps modify this module in-box and testing the
# changes, to avoid having to create a package file each time
self._debugNoInstallMode=False
class InstallManager(object):
"""
This is the main install manager
"""
def __init__ (self, logger, product, sysVarDir, sysRunDir, dataVarDir, rpmsDir,
appDir, appDirReal, bootDevPath, grubConf, prevRfsDir=None, vitalDir=None, systemRoot="/", useFakeChrootMode=False):
"""
product : Name of product we handle, e.g. 'qb'.
sysVarDir : path to sys dir relative to root, e.g. '/opt/qb/sys/oscar/install/0/var'.
sysRunDir : path to sys dir relative to root, e.g. '/opt/qb/sys/oscar/install/0/run'.
dataVarDir : path to data dir relative to root, e.g. '/opt/qb/data/oscar/install/0/var'.
rpmsDir : path to rpms dir relative to root, e.g. '/opt/qb/rpms'.
appDir : path to app dir relative to root, e.g. '/opt/qb/app'.
appDirReal : real path to app dir relative to root, e.g. '/opt/qb/sys/rfs/sys/0/var/app'.
bootDevPath : Path to boot device, e.g. '/boot'
grubConf : name of grub.conf file relative to boot device, e.g. 'grub/grub.conf'.
systemRoot : For unit tests, specifies location of the file-system root.
useFakeChrootMode : For unit tests, True to use fakechroot instead of chroot (via rpm --root or yum --installroot)
prevRfsDir : Location to copy some RFS data to during OS install
vitalDir : Path to vital dir.
"""
self._raiseIfNotAbsPath(systemRoot, "systemRoot")
self._log = logger.createLogger("sys-install", "install-manager")
self._log.setInstance(product)
self._utils = Utils(self._log)
self._product = product
self._bootDevPath = bootDevPath
self._grubConf = grubConf
self._systemRoot = systemRoot
self._useFakeChrootMode = useFakeChrootMode
self._vitalDir = vitalDir
self._rpmsDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(rpmsDir))
self._sysVarDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(sysVarDir))
self._sysRunDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(sysRunDir))
self._dataVarDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(dataVarDir))
self._logDir=os.path.join(self._dataVarDir, 'log')
self._stateDir = os.path.join(self._sysVarDir, "state")
self._yumConfActiveRfsDir = os.path.join(self._sysVarDir, "yum-conf-active-rfs")
self._yumConfNewVersionDir = os.path.join(self._sysVarDir, "yum-conf-new-version")
self._appDir = os.path.join(systemRoot, self._chopLeadingSlashIfFound(appDir))
self._appDirReal = os.path.join(systemRoot, self._chopLeadingSlashIfFound(appDirReal))
self._installDir = os.path.join(self._sysVarDir,"sub-pkgs")
self._workDir=os.path.join(self._sysVarDir, 'work')
self._apiDir=os.path.join(self._sysVarDir, 'api')
self._startupScriptsDir=os.path.join(self._sysVarDir, 'startup')
self._startupScriptsPrevDir=os.path.join(self._startupScriptsDir, 'prev')
self._rpmKeysDir = os.path.join(self._sysVarDir, "rpm/keys")
self._tempDir = os.path.join(self._sysRunDir, "tmp")
self._installTempDir = os.path.join(self._tempDir, "sub-pkgs") # install-temp can be cleared between runs
self._subPkgManager = SubPkgManager(self._log)
self._subPkgManager.setDirs(self._systemRoot, self._installDir, self._installTempDir, self._rpmKeysDir, self._useFakeChrootMode)
self._stateFile = os.path.join(self._stateDir, product+"-state")
self._stateManager = StateManager(self._log)
self._stateManager.setFile(self._stateFile)
self._pilotHints=""
self._prevRfsDir = prevRfsDir
self._preparedPackageFilePath = None
self._log("init").info("InstallManager::__init__() called, self._product=%s, self._sysVarDir=%s, self._sysRunDir=%s, "\
"self._dataVarDir=%s, self._rpmsDir=%s, self._appDir=%s, self._bootDevPath=%s, "\
"self._grubConf=%s, self._prevRfsDir=%s, self._vitalDir=%s",
self._product, self._sysVarDir, self._sysRunDir, self._dataVarDir, self._rpmsDir,
self._appDir, self._bootDevPath, self._grubConf, self._prevRfsDir, self._vitalDir)
# If true, we will not actually install RPMs. This helps modify this module in-box and testing the
# changes, to avoid having to create a package file each time
self._debugNoInstallMode=False
def unit_test_initializeSystemFirstInstall (self, curVersion, curBuild, keys):
"""
Should be called once after first install to initialize the state file and keys.
Used only by unit tests. In real life, the extract_pkg script does this.
curVersion : Name of the current version
curBuild : Name of the current build
keys : listof paths to ascii armoured files with gpg keys for rpm validation
"""
self.initializeSystemFirstInstall(curVersion, curBuild)
# Import rpm keys
rpm=self._createRpmForActiveRfs()
for key in keys:
rpm.doImport(key)
def initializeSystemFirstInstall (self, curVersion, curBuild):
"""
Should be called once after first install to initialize the state file and keys.
curVersion : Name of the current version
curBuild : Name of the current build
"""
self._log("initial-set-cur-version").info("initialSetCurVersion() called, curVersion=%s", curVersion)
# Create initial dirs
self._utils.runCommandRaiseIfFail("mkdir -p "+self._stateDir)
# Set initial state
self._stateManager.initialSetCurVersion(self.combineVersion(curVersion, curBuild))
def init (self):
"""
Must be called after creating this object, to initialize it, on a system that has been initialized
(i.e. has a state file on disk)
"""
self._log("init").info("init() called")
# Make sure these dirs exist. If not, create them.
self._utils.runCommandRaiseIfFail("mkdir -p "+self._sysRunDir)
self._utils.runCommandRaiseIfFail("mkdir -p "+self._logDir)
self._utils.runCommandRaiseIfFail("mkdir -p "+self._workDir)
self._utils.runCommandRaiseIfFail("mkdir -p "+self._apiDir)
self._utils.runCommandRaiseIfFail("mkdir -p "+self._startupScriptsDir)
self._utils.runCommandRaiseIfFail("mkdir -p "+self._startupScriptsPrevDir)
#TODO(orens): catch InstallException that might be raised during load, reset state and restart oscar
self._log("init").info("init() loading state")
self._stateManager.load()
self._initPlatformBasic()
self._initSdUtils()
self._initBootUtils()
def isPendingRestart (self):
"""
Returns true if 'switch' was executed, we are not waiting for system to restart.
"""
return self._stateManager.isLocked()
def setPilotHints (self, hints):
self._log("setPilotHints").info("setPilotHints(hints=%s) called", hints)
self._pilotHints = hints
def startup (self, logDir = None):
"""
Must be called during startup of an installed system, i,e, upon every Oscar start.
Call this after callint init()
When logDir is passed, the start-up script writes its logs into that directory, otherwise default directory is used.
"""
self._log("startup").info("startup() called")
#TODO(orens): In the future, when we do not need to support upgrading from EFT versions,
# remove this call, and all the setVersionToActivateOnStartup() support from StateManager
self._stateManager.startup()
# Run all scripts from the startup dir, move them to 'prev' later
self._log("startup-running").info("startup(): Scanning directory %s for startup scripts", self._startupScriptsDir)
files=os.listdir(self._startupScriptsDir)
for file_ in files:
fullPath=os.path.join(self._startupScriptsDir, file_)
if logDir is not None:
startupCommand = fullPath + " --log-dir " + logDir
else:
startupCommand = fullPath
# Do not run .tmp files or directories :-)
if (not fullPath.endswith(".tmp")) and os.path.isfile(fullPath):
self._log("startup-running").info("startup(): running startup script '%s'", startupCommand)
# If script fails, raise and get out of here. This should fail oscar start
self._utils.runCommandRaiseIfFail(startupCommand)
# Move used script to 'prev' dir, to lay there forever shamed.
self._utils.runCommandRaiseIfFail("mv -f %s %s" % (fullPath, self._startupScriptsPrevDir))
def getActiveVersionAndBuild (self):
"""
Returns the current active (version, build)
"""
self._log("get-active-version").info("getActiveVersionAndBuid() called")
versionAndBuild=self._stateManager.getActiveVersion()
(version_, build) = self.splitVersion(versionAndBuild)
self._log("get-active-version").info("getActiveVersionAndBuid() got %s, returning %s, %s",
versionAndBuild, version_, build)
return (version_, build)
def getReadyVersion (self):
"""
Returns (version, build) of currently ready version.
If no version is ready, returns None
"""
versionCombined = self._stateManager.getReadyVersion()
if versionCombined==None:
return None
(version_, build) = self.splitVersion(versionCombined, raiseIfInvalid=True)
return (version_, build)
def getReadyVersion2 (self):
"""
Returns (version, build, switch type, install direction) of currently ready version.
If no version is ready, returns None
"""
versionCombined = self._stateManager.getReadyVersion()
if versionCombined==None:
return None
switchType = self._stateManager.getSwitchType(versionCombined)
(version_, build) = self.splitVersion(versionCombined, raiseIfInvalid=True)
return (version_, build, switchType)
def getStateManager (self):
return self._stateManager
def getPreparedPackageFilePath (self):
return self._preparedPackageFilePath
def isPackageValid (self, fileName):
"""
Checks that an RPM file is a signed, and contains a proper version for our product.
Returns None if invalid, or (version, build) if valid
"""
self._log("is-valid").info("isPackageValid(fileName=%s) called", fileName)
if not self._isPackageSigned(fileName):
self._log("is-valid").warning("isPackageValid() returning false, fileName=%s not signed", fileName)
return None
versionCombined = self._subPkgManager.findVersionInFile(self._product, fileName)
if versionCombined is None:
self._log("is-valid").warning("isPackageValid() returning false, no version of product %s found in it", self._product)
return None
(version_, build) = self.splitVersion(versionCombined)
if version_ is None:
self._log("is-valid").warning("isPackageValid() returning false, no version of product %s found in it", self._product)
return None
return (version_, build)
def addSubPkgs (self, fileName):
"""
Adds a package file to the repos. Raises InvalidPackageFileException if not a valid package file for this product
Returns (version, build)
"""
self._log("add").info("addSubPkgs(fileName=%s) called", fileName)
self._raiseIfNotAbsPath(fileName, "fileName")
valid=self._isPackageSigned(fileName)
if not valid:
self._log("add-bad-rpm").error("addSubPkgs() got bad package file, not properly signed", self._product)
raise InvalidPackageFileException()
versionCombined = self._subPkgManager.findVersionInFile(self._product, fileName)
if versionCombined is None:
self._log("add-bad-version").error("addSubPkgs() got bad package file, no version of product %s found in it", self._product)
raise InvalidPackageFileException()
(version_, build) = self.splitVersion(versionCombined)
if version_ is None:
self._log("add-bad-version").error("addSubPkgs() got versionInFile '%s', cannot split.", versionCombined)
raise InvalidPackageFileException()
# Check if version is active or ready. If yes, raise exception
self._raiseIfVersionActiveOrReady(versionCombined)
# Extract package file into sub-pkgs dir
(subPkgs, newSubPkgs)=self._subPkgManager.add(fileName)
self._log("add-got-subpkgs").info("addSubPkgs() returning newSubPkgs=%s", newSubPkgs)
self._stateManager.addVersion(versionCombined, subPkgs=subPkgs, subPkgsToRemove=newSubPkgs)
# Remember the file path
self._preparedPackageFilePath = fileName
self._log("add").info("addSubPkgs() returning version=%s, build=%s", version_, build)
return (version_, build)
def addSubPkgsFromHttp (self, repoHttpAddr, version_, build):
"""
Adds a package file to the repos. Raises InstallException if package not found.
"""
self._log("add-from-http").info("addSubPkgsFromHttp(repoHttpAddr=%s, version_=%s, build=%s) called", repoHttpAddr, version_, build)
versionCombined = self.combineVersion(version_, build)
if versionCombined is None:
self._log("add-from-http-bad-version").error("addSubPkgsFromHttp() got bad version(=%s) or build(=%s) parameters", version_, build)
raise InstallException("Invalid version '%s', build '%s' specificaion" % (version_, build))
# Check if version is active or ready. If yes, raise exception
self._raiseIfVersionActiveOrReady(versionCombined)
packageName = "%s-%s" % (self._product, versionCombined)
self._log("add-from-http-package").info("addSubPkgsFromHttp(): installing package %s", packageName)
# Extract package file into sub-pkgs dir
(subPkgs, newSubPkgs)=self._subPkgManager.add(httpRepo=repoHttpAddr, packageName=packageName)
self._log("add-from-http-got-subpkgs").info("addSubPkgsFromHttp() got newSubPkgs=%s", newSubPkgs)
self._stateManager.addVersion(versionCombined, subPkgs=subPkgs, subPkgsToRemove=newSubPkgs)
self._log("add-from-http-done").info("addSubPkgsFromHttp() done")
def removeVersionAndSubPkgs (self, version_, build, removeSubPackages=True):
"""
Removes sub-pkgs added by a version during 'prepare', and the version itself.
Version must have state 'on-disk'
"""
self._log("remove-subpkgs").info("removeVersionAndSubPkgs(version=%s, build=%s) called", version_, build)
versionCombined = self.combineVersion(version_, build)
versionState = self._stateManager.getVersionState(versionCombined)
if (versionState != StateManager.kStateOnDisk):
msg = "removeVersionAndSubPkgs(): Version %s is not on-disk, it is in state '%s'" % (versionCombined, versionState)
self._log("remove-subpkgs").error(msg)
raise InstallException(msg)
if removeSubPackages:
subPkgsToRemove = self._stateManager.getSubPkgsToRemoveOnRemoval(versionCombined)
self._log("remove-subpkgs").info("Removing sub-packages: %s", subPkgsToRemove)
if len(subPkgsToRemove):
self._subPkgManager.removeSubPackages(subPkgsToRemove)
self._log("remove-subpkgs").info("Removing version %s from state", versionCombined)
self._stateManager.removeVersion(versionCombined)
def removeUnneededVersionsFromDisk (self):
"""
Removes old versions from disk.
Scans all versions maintained in the state, and for each version which is 'on-disk', if it is not needed (i.e.
for a possible rollback command), removes sub-packages and RPMs installed by it and not required by the active versions
"""
self._log("remove-undeeded-vers-called").info("removeUnneededVersionsFromDisk() called")
# Find things needed by the current active and ready versions
# Currently no rollback support, hence nothing else is needed.
versions = self._stateManager.getVersions()
versionsToRemove=[]
versionsToKeep=[]
for ver in versions:
state=self._stateManager.getVersionState(ver)
# TODO(orens): When we add rollback support, keep also the 'on-disk' version we may rollback into.
if state==StateManager.kStateOnDisk:
versionsToRemove.append(ver)
else:
versionsToKeep.append(ver)
# Find list of RPMs and list of subpackages we must keep
(rpmsToKeep, subPkgsToKeep)=self._getRpmsAndSubPkgsOfVersions(versionsToKeep)
# Find list of RPMs used by versions we remove
(rpmsToRemove, subPkgsToRemove)=self._getRpmsAndSubPkgsOfVersions(versionsToRemove)
# The crucial moment
rpmsToRemove -= rpmsToKeep
subPkgsToRemove -= subPkgsToKeep
# Filter rpms in order to check which ones are not installed- only installed ones should be removed
self._log("ruv-filtering1").info("removeUnneededVersionsFromDisk(): Filtering RPMs %s for not installed packages", rpmsToRemove)
rpm=self._createRpmForActiveRfs()
rpmsToRemove = filter(rpm.isInstalled, rpmsToRemove)
# Check if pilot wants to modify RPM list
activeVersion = self._stateManager.getActiveVersion()
self._log("ruv-filtering3").info("removeUnneededVersionsFromDisk(): activeVersion is %s", activeVersion)
(pilot, _) = self._createPilot(activeVersion, None)
if hasattr(pilot, "hookModifyRpmsForRemovalOld"):
rpmsToRemove=pilot.hookModifyRpmsForRemovalOld(rpmsToRemove)
self._log("ruv-filtering5").info("removeUnneededVersionsFromDisk(): pilot hookModifyRpmsForRemovalOld() returned %s", rpmsToRemove)
else:
self._log("ruv-filtering6").info("removeUnneededVersionsFromDisk(): pilot hookModifyRpmsForRemovalOld() not implemented")
# We do not need to filter sub-pkgs for existance - only existing ones would be removed
self._log("ruv-removing-ver").info("removeUnneededVersionsFromDisk(): Removing RPMs %s", rpmsToRemove)
self._log("ruv-removing-ver").info("removeUnneededVersionsFromDisk(): Removing sub-pkgs %s", subPkgsToRemove)
if len(rpmsToRemove)>0:
yum=self._createYumWithAllRepos()
yum.doRemove(" ".join(rpmsToRemove))
self._subPkgManager.removeSubPackages(subPkgsToRemove)
self._log("ruv-removing-ver").info("removeUnneededVersionsFromDisk(): Removing versions %s from state", versionsToRemove)
for ver in versionsToRemove:
self._stateManager.removeVersion(ver)
def _getRpmsAndSubPkgsOfVersions(self, versions):
"""
Returns a tuple with two sets: (rpmsOfVersions, subPkgsOfVersions)
"""
rpmsOfVersions=set()
subPkgsOfVersions=set()
for ver in versions:
self._log("get-rpms-and-subpkgs").info("_getRpmsAndSubPkgsOfVersions() Handling version %s", ver)
leaders=self._stateManager.getLeaders(ver)
self._log("get-rpms-and-subpkgs").info("_getRpmsAndSubPkgsOfVersions(): leaders=%s", leaders)
for leader in leaders.values():
rpmsOfVersions.add(leader)
subPkgs=self._stateManager.getSubPkgs(ver)
self._log("get-rpms-and-subpkgs").info("_getRpmsAndSubPkgsOfVersions(): subPkgs=%s", subPkgs)
for subPkg in subPkgs:
subPkgsOfVersions.add(subPkg)
# The pilot of a version is always called like this:
rpmsOfVersions.add(self.calcPilotName(ver))
# Expand set by adding all required packages
rpmsOfVersions=self._expandRpmsUsingReqires(rpmsOfVersions)
self._log("get-rpms-and-subpkgs").info("_getRpmsAndSubPkgsOfVersions(): Returning rpmsOfVersions=%s", rpmsOfVersions)
self._log("get-rpms-and-subpkgs").info("_getRpmsAndSubPkgsOfVersions(): Returning subPkgsOfVersions=%s", subPkgsOfVersions)
return (rpmsOfVersions, subPkgsOfVersions)
def _expandRpmsUsingReqires (self, packages):
"""
Gets an iterable with a list of packages (Some may not be installed it is OK).
Returns a set with the same packages, plus all installed packages required by these installed packages
"""
self._log("expand-requires").info("_expandRpmsUsingReqires(packages=%s) called, ", packages)
rpm=self._createRpmForActiveRfs()
doIt=True
ret=set(packages)
while doIt:
self._log("expand-requires").info("_expandRpmsUsingReqires() ret=%s", ret)
doIt=False
addedPackages=set()
for package in packages:
requires=rpm.doListRequires(package)
for (capName, capCondition, capVersion) in requires:
# Ignore 'rpmxxx' capabilities
if capName.startswith('rpm'):
continue
# Ignore capabilities which are not '='.
if capCondition != '=':
continue
addedPackages.add(capName+'-'+capVersion)
# Found something new ? Add to result and continue iterating
self._log("expand-requires").info("_expandRpmsUsingReqires() addedPackages=%s, ret=%s", addedPackages, ret)
newPackages = addedPackages-ret
self._log("expand-requires").info("_expandRpmsUsingReqires() newPackages=%s", newPackages)
if len(newPackages) > 0:
doIt=True
ret |= newPackages
packages = newPackages
self._log("expand-requires").info("_expandRpmsUsingReqires() returning %s", ret)
return ret
def prepare (self, version_, build, usePilotVersion=None):
"""
Prepares a version for installation.
If a version is already prepared, raises InstallException().
usePilotVersion: If != None, sets pilot version to be used instead of auto-selecting.
"""
self._log("prepare").info("prepare(version=%s, build=%s) called", version_, build)
if self._stateManager.getReadyVersion() != None:
msg="prepare(): A version is ready"
self._log("prepare").error(msg)
raise InstallException(msg)
if build == kFakeBuildNumber:
msg="prepare(): Invalid build number. Build '%s' is reserved for fake builds" % kFakeBuildNumber
self._log("prepare").error(msg)
raise InstallException(msg)
newVersion=self.combineVersion(version_, build)
activeVersion=self._stateManager.getActiveVersion()
self._installPilot(newVersion)
self._log("prepare").info("Creating pilot for new version %s", newVersion)
(newVersionPilot, newVersionPs) = self._createPilot(newVersion, newVersion)
self._log("prepare").info("Creating pilot for active version %s", activeVersion)
(activeVersionPilot, activeVersionPs) = self._createPilot(activeVersion, newVersion)
# Teach each pilot services about the other pilot
newVersionPs.setOtherPilot(activeVersionPilot)
activeVersionPs.setOtherPilot(newVersionPilot)
if usePilotVersion != None:
self._log("prepare").info("Creating pilot for version %s, manually selected", usePilotVersion)
if (usePilotVersion != newVersion) and (usePilotVersion != activeVersion):
msg="Manually selected pilot should be one of %s or %s, not %s" % (newVersion, activeVersion, usePilotVersion)
self._log("prepare").error("prepare(): %s", msg)
raise InstallException(msg)
selectedVersion=usePilotVersion
else:
self._log("prepare").info("Running pilot selection")
# old pilot selection logic uses 'selectPilot'
# new pilot selection logic uses 'getPilotGeneration'
# If new version pilot supports new logic, use it.
# If new version pilot does not support new logic, then this is downgrade, we choose our pilot.
if hasattr(newVersionPilot, "getPilotGeneration"):
newPilotGeneration=newVersionPilot.getPilotGeneration()
activePilotGeneration=activeVersionPilot.getPilotGeneration()
self._log("prepare").info("prepare(): pilot selection: newVersion=%s, activeVersion=%s, newPilotGeneration=%s, activePilotGeneration=%s",
newVersion, activeVersion, newPilotGeneration, activePilotGeneration)
if newPilotGeneration > activePilotGeneration:
selectedVersion = newVersion
elif newPilotGeneration < activePilotGeneration:
selectedVersion = activeVersion
else: # Pilot generations are equal, choose according to version numbers
selectedVersion=self._selectLatestVersion(newVersion, activeVersion)
else: # Downgrading, choose our pilot
selectedVersion = activeVersion
if selectedVersion == newVersion:
pilot = newVersionPilot
elif selectedVersion == activeVersion:
pilot = activeVersionPilot
else:
msg="Bad pilot: newVersion=%s, activeVersion=%s, selectedVersion=%s" % (newVersion, activeVersion, selectedVersion)
self._log("prepare").error("prepare(): pilot selection: %s", msg)
raise InstallException(msg)
# Version is selected, We have a pilot, let's take off
self._log("prepare").info("prepare(): Selected pilot version is %s", selectedVersion)
self._stateManager.setPilotName(newVersion, selectedVersion)
rpm = self._createRpmForActiveRfs()
installedRpmsBefore=rpm.doGetInstalledPackages()
self._log("prepare").info("prepare(): Installed RPMs before prepare() are: %s", installedRpmsBefore)
# As last, Run pilot 'prepare' function !
# Pilot is executed in try block. In case of failure, we clear pilot name.
self._log("prepare").info("prepare(): running pilot...")
try:
# Old pilots do not have doPrepare2(), but we should never call old pilots from new code,
# because the active version pilot should win.
pilot.doPrepare2(newVersion, newVersionPilot, activeVersionPilot)
except:
self._stateManager.setPilotName(newVersion, None)
raise
installedRpmsAfter=rpm.doGetInstalledPackages()
self._log("prepare").info("prepare(): Installed RPMs after prepare() are: %s", installedRpmsBefore)
newRpms = sorted(list(set(installedRpmsAfter)-set(installedRpmsBefore)))
self._log("prepare").info("prepare(): New RPMs: %s", newRpms)
self._stateManager.setRpmsToRemoveOnRemoval(newVersion, newRpms)
# Pilot prepare went ok - time to mark this version as 'ready' !
self._log("prepare").info("prepare(): pilot did good, marking version as ready")
self._stateManager.setReadyVersion(newVersion)
def cancel (self, version_, build):
self._log("cancel").info("cancel(version=%s, build=%s) called", version_, build)
versionCombined = self.combineVersion(version_, build)
versionState = self._stateManager.getVersionState(versionCombined)
if (versionState != StateManager.kStateReady):
msg = "cancel(): Version %s is not ready, it is in state '%s'" % (versionCombined, versionState)
self._log("cancel").error(msg)
raise InstallException(msg)
self._log("cancel").info("Creating pilot for version %s", versionCombined)
(pilot,_)=self._createPilot(self._stateManager.getPilotName(versionCombined), versionCombined)
if hasattr(pilot, "hookCancelBeforeRemove"):
self._log("cancel").info("Calling pilot hookCancelBeforeRemove()")
pilot.hookCancelBeforeRemove(versionCombined)
rpmsToRemove = self._stateManager.getRpmsToRemoveOnRemoval(versionCombined)
rpm = self._createRpmForActiveRfs()
self._log("cancel").info("Removing rpms: %s", rpmsToRemove)
if len(rpmsToRemove):
rpm.doRemove(" ".join(rpmsToRemove))
if hasattr(pilot, "hookCancelAfterRemove"):
self._log("cancel").info("Calling pilot hookCancelAfterRemove()")
pilot.hookCancelAfterRemove(versionCombined)
self._log("cancel").info("Changing state of version %s to 'on-disk'", versionCombined)
self._stateManager.setVersionStateOnDisk(versionCombined)
def switch (self, version_, build):
"""
version_, build are parameters of the ready version
On success, returns pilot doSwitch() return code.
On failure, raises something
"""
self._log("switch-called").info("switch(version=%s, build=%s) called", version_, build)
versionCombined = self.combineVersion(version_, build)
versionState = self._stateManager.getVersionState(versionCombined)
if (versionState != StateManager.kStateReady):
msg = "cancel(): Version %s is not ready, it is in state '%s'" % (versionCombined, versionState)
self._log("cancel").error(msg)
raise InstallException(msg)
self._log("switch").info("switch() creating pilot for version %s", versionCombined)
pilotName=self._stateManager.getPilotName(versionCombined)
(pilot,_)=self._createPilot(pilotName, versionCombined)
self._log("switch").info("switch() forgetting things to remove on removal for version %s", versionCombined)
self._stateManager.setSubPkgsToRemoveOnRemoval(versionCombined, None)
self._stateManager.setRpmsToRemoveOnRemoval(versionCombined, None)
# Get params needed to determine which user-log message to emit, and emit it
activeVersion=self._stateManager.getActiveVersion()
switchType = self._stateManager.getSwitchType(versionCombined)
installDirection = self._stateManager.getInstallDirection(versionCombined)
self._log("switch").info("switch() got switchType='%s', installDirection='%s'", switchType, installDirection)
if (switchType == "OS Restart") and (installDirection == "upgrade"):
a.infra.process.logUserMessage(a.api.user_log.msg.install.OsUpgrade(activeVersion, versionCombined))
elif (switchType == "OS Restart") and (installDirection == "downgrade"):
a.infra.process.logUserMessage(a.api.user_log.msg.install.OsDowngrade(activeVersion, versionCombined))
elif (switchType == "Application Restart") and (installDirection == "upgrade"):
a.infra.process.logUserMessage(a.api.user_log.msg.install.AppUpgrade(activeVersion, versionCombined))
elif (switchType == "Application Restart") and (installDirection == "downgrade"):
a.infra.process.logUserMessage(a.api.user_log.msg.install.AppDowngrade(activeVersion, versionCombined))
self._log("switch").info("switch() calling pilot doSwitch(%s)", versionCombined)
# Returned value from pilot should signal caller something (Probably exit code).
ret = pilot.doSwitch(versionCombined)
# State is now locked. Let's check it, in case pilot was sloppy.
if not self._stateManager.isLocked():
msg="ERROR: switch() expecting state to be locked after pilot doSwitch()"
self._log("switch").error(msg)
raise InstallException(msg)
return ret
def verify (self):
"""
Verifies all installed RPMs.
Returns list of problems, empty list if OK
"""
self._log("verify-called").info("verify() called")
rpm=self._createRpmForActiveRfs()
problems = rpm.doVerifyAllInstalled()
self._log("verify-returning").info("verify() returning %s", problems)
# We skip problems about some files which normally change after being installed
filteredProblems = []
for problem in problems:
# /etc and /var files onbviously should not be checked
if " /etc/" in problem:
continue
if " /var/" in problem:
continue
# .ini and .cfg files in /opt/dell seem to be changing
if (" /opt/dell/" in problem) and (problem.endswith(".ini") or problem.endswith(".cfg")):
continue
# .pyc files in Oscar seem to be changing
if (" /opt/qb/rpms/oscar-" in problem) and problem.endswith(".pyc"):
continue
# We over-ride firmware files when upgrading the kernel
if " /lib/firmware/" in problem:
continue
# Handle a few specific files
if (" /opt/MegaRAID/MegaCli/MegaCli" in problem) or (" /root/.bash_profile" in problem):
continue
filteredProblems.append(problem)
return filteredProblems
def getRpmsDir (self):
return self._rpmsDir
def getAppDir (self):
return self._appDir
def getAppNextDir (self):
return self._appDir+".next"
def getAppPrevDir (self):
return self._appDir+".prev"
def getAppDirReal (self):
return self._appDirReal
def getBootDevPath (self):
return self._bootDevPath
def getGrubConf (self):
return self._grubConf
def getVitalDir (self):
return self._vitalDir
def getSubPkgManager (self):
return self._subPkgManager
def debugSetNoInstallMode (self):
self._log("debug-no-install-mode").notice("debugSetNoInstallMode() called, NO RPMS WILL BE INSTALLED")
self._debugNoInstallMode=True
def unit_test_ExtractPackageFile (self, fileName):
self._subPkgManager.add(r2pmFile=fileName)
def unit_test_InstallPilot (self, version_, build):
versionCombined=self.combineVersion(version_, build)
self._installPilot(versionCombined)
def unit_test_InstallRpm (self, name):
yum=self._createYumWithAllRepos()
self._log("debug-install-rpm").info("debugInstallRpm(): installing %s", name)
if not self._debugNoInstallMode:
yum.doInstall(name)
def unit_test_GetInstallTempDir (self):
"""Helper for unit tests"""
return self._installTempDir
def unit_test_GetStateFile (self):
"""Helper for unit tests"""
self._log("get-state-file").info("unit_test_GetStateFile(): returning %s", self._stateFile)
return self._stateFile
def _initPlatformBasic (self):
self._log("init-platform-basic").info("initializing the platform-basic")
self._platformBasic = a.infra.process.getPlatformBasicDataOrCrash()
def _initSdUtils (self):
self._log("init-sd-utils").info("initializing the sd-utils")
self._sdUtils = SdUtils(self._log)
def _initBootUtils (self):
self._log("init-boot-utils").info("initializing the boot-utils")
self._bootUtils = BootUtils.s_getFromOsefOrCrash(a.infra.process.getOsef())
def _installPilot(self, versionCombined):
yum=self._createYumWithAllRepos()
pilotPackageName = self.calcPilotName(versionCombined)
self._log("install-pilot").info("_installPilot(): pilotPackageName=%s", pilotPackageName)
if not self._debugNoInstallMode:
yum.doInstall(pilotPackageName)
def _createPilot (self, versionCombined, newVersionCombined):
"""
Returns (pilot, pilotServices)
versionCombined - name of pilot to create
newVersionCombined - name of version being installed ('v-b')
"""
self._log("create-pilot").info("_createPilot() called, versionCombined = %s", versionCombined)
pilotRpmDirName = self.calcPilotName(versionCombined)
pilotDirName = os.path.join(self._rpmsDir, pilotRpmDirName)
pilotFileName = os.path.join(pilotDirName, "pilot.py")
self._log("create-pilot").info("_createPilot(): pilotFileName = %s", pilotFileName)
if not os.path.exists(pilotFileName):
self._log("create-pilot").error("_createPilot() Could not find pilot file %s", pilotFileName)
raise InstallException("Could not find pilot file %s" % pilotFileName)
# Keep a copy of sys.path
oldSysPath=list(sys.path)
try:
sys.path.append(pilotDirName)
self._log("create-pilot").info("_createPilot(): about to import, sys.path=%s", sys.path)
import pilot
reload(pilot)
p=pilot.Pilot()
except:
self._log("create-pilot").error("_createPilot(): Got exception when importing and creating pilot")
raise
finally:
# Restore old sys.path
sys.path=oldSysPath
self._log("create-pilot").info("_createPilot(): Restored sys.path to %s", sys.path)
self._log("create-pilot").info("_createPilot(): creating pilot services, pilotDirName=%s", pilotDirName)
pilotServices=self._createPilotServices(versionCombined, pilotDirName, newVersionCombined)
p.setPilotServices(pilotServices)
self._log("create-pilot-returning").info("_createPilot() returning pilot")
return (p, pilotServices)
def calcPilotName (self, versionCombined):
pilotName="pilot-%s-%s" % (self._product, versionCombined)
return pilotName
def _isPackageSigned (self, fileName):
"""
Checks that an RPM file is a signed
"""
rpm=self._createRpmForActiveRfs()
return rpm.doValidateRpm(fileName)
def combineVersion (self, version_, build):
"""Combines version and build into a single string "v-b". version and buil cannot contain '-' chars. """
if version_.find("-") != -1:
self._log("combine-version-bad").error("combineVersion() got version='%s', invalid", version_)
raise InstallException("combineVersion(): bad version '%s'" % version_)
if build.find("-") != -1:
self._log("combine-version-bad").error("combineVersion() got build='%s', invalid", build)
raise InstallException("combineVersion(): bad build '%s'" % build)
return version_+"-"+build
def splitVersion (self, versionCombined, raiseIfInvalid=False):
"""
Splits versionCombined into a version and a build. versionCombined must contain exactly 1 '-' char.
Returns (version, build), or (None, None) if versionCombined is invalid
if raiseIfInvalid is True, then instead of returning (None, None), raises InstallException
"""
if versionCombined.count("-") != 1:
self._log("version-split-bad").error("splitVersion() got '%s', cannot split", versionCombined)
if raiseIfInvalid:
raise InstallException("splitVersion(): bad versionCombined '%s'" % versionCombined)
return (None, None)
(version_,build) = versionCombined.split("-")
return (version_,build)
def _createPilotServices (self, pilotVersion, pilotDir, newVersionCombined):
self._log("create-pilot-services").info("_createPilotServices(): Called, pilotVersion = %s, newVersionCombined = %s",
pilotVersion, newVersionCombined)
rpmActiveRfs=self._createRpmForActiveRfs()
yumActiveRfs=self._createYumWithAllRepos()
yumNewVersionOnly=self._createYumWithAllRepos(repoFilter=newVersionCombined)
self._log("create-pilot-services").info("_createPilotServices(): creating yum.conf for active RFS and yum.conf "\
"for new version %s", newVersionCombined)
yumConfActiveRfs=yumActiveRfs.createYumConfAt(self._yumConfActiveRfsDir)
yumConfNewVersionOnly=yumNewVersionOnly.createYumConfAt(self._yumConfNewVersionDir)
self._log("create-pilot-services").info("_createPilotServices(): got yum.conf at %s, yum.conf for new versiom only at %s",
yumConfActiveRfs, yumConfNewVersionOnly)
pilotServices=PilotServices(self._log, self._stateManager, self, pilotVersion, pilotDir, self._logDir, self._workDir, self._apiDir,
self._startupScriptsDir, rpmActiveRfs, yumActiveRfs, yumConfActiveRfs, yumConfNewVersionOnly,
self._prevRfsDir, self._sdUtils, self._bootUtils)
pilotServices.setPilotHints(self._pilotHints)
return pilotServices
def _createRpmForActiveRfs (self):
rpm=Rpm(self._log)
rpm.setRoot(self._systemRoot, useFakeChrootMode=self._useFakeChrootMode)
return rpm
def _createYumWithAllRepos (self, repoFilter=None):
"""
Returns a yum object configured to see all repos
if repoFilter != None, yum is configured to see only repos which contain the subs-string repoFilter:
"""
self._log("create-yum-with-all-repos").info("createYumWithAllRepos() called, repoFilter=%s", repoFilter)
if self._useFakeChrootMode:
yum=Yum(self._log, self._tempDir.replace(self._systemRoot, ""))
yum.setRoot(self._systemRoot, useFakeChrootMode=True)
else:
yum=Yum(self._log, os.path.join(self._tempDir, "yum"))
self._subPkgManager.addReposToYum(yum, repoFilter)
return yum
def _chopLeadingSlashIfFound (self, path_):
if path_.startswith('/'):
return path_[1:]
return path_
def _selectLatestVersion(self, version1, version2):
# Split versions to version name and build number
self._log("select-latest-version").info("_selectLatestVersion() called, version1=%s, version2=%s", version1, version2)
# Versions must look like this: "v-b"
vb1=version1.split('-')
if len(vb1) != 2:
raise InstallException("Invalid version number: %s is invalid" % version1)
vb2=version2.split('-')
if len(vb2) != 2:
raise InstallException("Invalid version number: %s is invalid" % version2)
ver1=vb1[0]
b1=vb1[1]
ver2=vb2[0]
b2=vb2[1]
# A version is official if it is made from numbers and dots only
onlyDigitsAndDots="^[\d\.]+$"
ver1IsOfficial=bool(re.match(onlyDigitsAndDots, version1))
ver2IsOfficial=bool(re.match(onlyDigitsAndDots, version2))
# Official versions beat private versions
if ver1IsOfficial != ver2IsOfficial:
if ver1IsOfficial:
return version1
return version2
# Now we have either two official versions, or two private versions.
if ver1 != ver2:
# Versions are not identical. Split version into elements
v1=ver1.split(".")
v2=ver2.split(".")
len1=len(v1)
len2=len(v2)
# Numerically compare the numbers, as long as both versions have them
minLen=min(len1, len2)
for i in range(minLen):
# Both version elements are the same ? no need to compare
if v1[i] == v2[i]:
continue
# Need to compare version elements. Convert to integers.
try:
ve1=int(v1[i])
ve2=int(v2[i])
except:
# Not an integer ? Cannot compare. Choose first version arbitrarily
return version1
# Different numbers ? We have a winner !
if ve1<ve2:
return version2
if ve1>ve2:
return version1
# Note that we may get here despite the 'if' that prevented comparing the same strings.
# For example, 04 and 4 are the same.
# Out of the loop with no decision ? Let's see if the version lengths want to talk
#
# Longer length wins: 2.1.0.0 is 'later' than 2.1.0
if len1<len2:
return version2
if len1>len2:
return version1
# If we are here, then versions are not identical, but all their components are the same
# AND they are the same length ? This could happen with versions like 2.04 vs 2.4.
# Choose arbitrarily
return version1
# Versions are the same. We need to decide based on build numbers
# Make sure build number is an integer
try:
b1n=int(b1)
b2n=int(b2)
except:
raise InstallException("Invalid build number: Both '%s' and '%s' should be numeric" % (b1, b2))
# Decide according to the build number
if b1n<b2n:
return version2
return version1
def _raiseIfVersionActiveOrReady(self, versionCombined):
# Do not allow adding an existing version
curState=self._stateManager.getVersionState(versionCombined)
if curState == StateManager.kStateActive:
self._log("add-existing-version").error("_raiseIfVersionActiveOrReady() Found existing active version '%s'", versionCombined)
raise InstallException("Version %s is the currently active version" % versionCombined)
if curState == StateManager.kStateReady:
self._log("add-existing-version").error("_raiseIfVersionActiveOrReady() Found existing ready version '%s'", versionCombined)
raise InstallException("Version %s is already ready" % versionCombined)
def _raiseIfAbsPath (self, pathToCheck, name):
if pathToCheck.startswith(os.path.sep):
raise ValueError("Parameter '%s' must not start with '/', value given is '%s'" % (name, pathToCheck))
def _raiseIfNotAbsPath (self, pathToCheck, name):
if not pathToCheck.startswith(os.path.sep):
raise ValueError("Parameter '%s' must start with '/', value given is '%s'" % (name, pathToCheck))
class SubPkgManager(object):
def __init__ (self, logger):
self._log=logger.createLogger("sys-install","sub-pkg-mng")
self.utils=Utils(self._log)
self.useFakeChrootMode=False
self.rootDir=None
self.subPkgDir=None
self.subPkgTempDir=None
def setDirs (self, rootDir, subPkgDir, subPkgTempDir, rpmKeysDir, useFakeChrootMode=False):
self._raiseIfNotAbsPath(subPkgDir, "subPkgDir")
self._raiseIfNotAbsPath(subPkgTempDir, "subPkgTempDir")
self.rootDir=rootDir
self.subPkgDir=subPkgDir
self.subPkgTempDir=subPkgTempDir
self.rpmKeysDir=rpmKeysDir
self.useFakeChrootMode=useFakeChrootMode
self.infoDir=os.path.join(self.subPkgDir, "info")
# During first install, sub-packages dir may not exist
if not os.path.exists(self.subPkgDir):
self.utils.runCommandRaiseIfFail("mkdir -p "+self.subPkgDir)
def findVersionInFile (self, product, r2pmFile):
"""
Expects an r2pm file with exactly one sub-pkg called 'qb-VVV-BBB'.
File must be below root dir
Returns version found in r2pm file (i.e. the string VVV-BBB).
If not exactly one such sub-pkg is found, returns None (="Invalid package file")
"""
self._log("find-version-called").info("findVersionInFile() called, product=%s, r2pmFile=%s", product, r2pmFile)
self._raiseIfNotAbsPath(r2pmFile, "r2pmFile")
# Find version
rpm=Rpm(self._log)
if self.useFakeChrootMode:
rpm.setRoot(self.rootDir, self.useFakeChrootMode)
r2pmFile=self._getFilePathFromRoot(r2pmFile)
files=rpm.doListFiles(r2pmFile)
self._log("find-version-files").info("findVersionInFile(): files=%s", files)
version_=None
count_=0
for file_ in files:
mo=re.match("^/info/"+product+"-(.*)\.info$",file_)
if mo != None:
version_=mo.group(1)
count_+=1
if count_ != 1:
self._log("find-version-bad-count").error("findVersionInFile(): File %s has %s sub-pkgs called "
"'%s-...', should have exactly one", r2pmFile, count_, product)
return None
return version_
def add (self, r2pmFile=None, httpRepo=None, packageName=None):
"""
Adds sub-packages, either from a r2pm file or from an http repo.
To install from file:
r2pmFile : file name
To install from an http repo:
httpRepo : repo address
packageName : Name of package containing the sub-packages
Returns a tuple (subPkgs, subPkgstoRemove).
subPkgs: list of sub-pkgs included in this version.
subPkgstoRemove: list of new sub-pkgs (Replaced sub-pkgs are not included in that list). These sub-pkgs
will be removed if 'prepare' operation is canceled
"""
self._log("add-called").info("add() called, r2pmFile=%s, httpRepo=%s, packageName=%s", r2pmFile, httpRepo, packageName)
# Check that our input makes sense
if (r2pmFile == None) == (httpRepo == None):
msg = "add(): bad parameters, must use either r2pmFile or httpRepo"
self._log("add-called-bad").error(msg)
raise InstallException(msg)
if r2pmFile != None:
self._raiseIfNotAbsPath(r2pmFile, "r2pmFile")
else:
if packageName == None:
msg = "add(): bad parameters, httpRepo must come with packageName"
self._log("add-called-bad").error(msg)
raise InstallException(msg)
# Clean the temp dir and create it
if not self.useFakeChrootMode:
# In ut fakechroot mode we do not remove this, because this dir has chroot soft links
self._cleanTempDir()
self.utils.runCommandRaiseIfFail("mkdir -p "+self.subPkgTempDir)
# Import RPM public keys
# In unit tests, it is impossible to get keys from outside of the chroot. Instead, the test does rpm --import
# in this chroot for us.
if not self.useFakeChrootMode:
rpm=Rpm(self._log)
rpm.setRoot(self.subPkgTempDir)
for keyFile in os.listdir(self.rpmKeysDir):
rpm.doImport(os.path.join(self.rpmKeysDir, keyFile))
# Check space - must have at least 4GB free
availableSize=self.utils.getAvailableSize(self.subPkgTempDir)
if availableSize < 4e9:
raise InstallException("Internal error: Not enough space on system disk")
# Read all existing sub-pkgs
oldInfoDict=self._readInfoFiles(self.infoDir)
self._log("add-sub-pkg").info("add(): Found old sub-pkgs %s", oldInfoDict)
oldSubPkgs=oldInfoDict.keys()
# Create temp dir for yum (for config file and such)
if not self.useFakeChrootMode:
yumTempDir=os.path.join(self.subPkgTempDir, "tmp")
else:
yumTempDir="/tmp"
# Extract r2pm into temp dir
try:
if r2pmFile != None:
# Install from file
removeFile=False
if not self.useFakeChrootMode:
r2pmNameToYum=r2pmFile
# In CentOS6.2, package files must end with '.rpm'
if not r2pmNameToYum.endswith('.rpm'):
r2pmNameToYum=os.path.join(self.subPkgTempDir, 'package.rpm')
self._log("add-sub-pkg-copy").info("add(): Copying %s to %s to have a file name that ends with .rpm", r2pmFile, r2pmNameToYum)
self.utils.runCommandRaiseIfFail("cp -f %s %s" % (r2pmFile, r2pmNameToYum))
removeFile=True
else:
self.utils.runCommandRaiseIfFail("cp "+r2pmFile+" "+self.subPkgTempDir)
r2pmNameToYum="/"+os.path.basename(r2pmFile)
yum=Yum(self._log, yumTempDir)
yum.setRoot(self.subPkgTempDir, useFakeChrootMode=self.useFakeChrootMode)
yum.doLocalInstall(r2pmNameToYum)
if removeFile:
self._log("add-sub-pkg-remove").info("add(): Removing file that was copied")
self.utils.runCommandRaiseIfFail("rm -f %s" % r2pmNameToYum)
else:
# Install from http repo
yum=Yum(self._log, yumTempDir)
yum.setRoot(self.subPkgTempDir, useFakeChrootMode=self.useFakeChrootMode)
yum.addHttpRepo("network", httpRepo)
yum.doInstall(packageName)
# Now, move/replace new sub-pkgs into subPkgDir.
# We do this according to a very specific sequence, so that in case we abort in the middle (power failure or whatever),
# A call to TBD() will restore things the way they were before this operation
# Todo(orens): TBD -> Real function
# Sequence is:
# 1. Copy all existing sub-packages to self.subPkgTempDir, except for those found there already (i.e. new copy
# is better tahn old copy)
# 2. Rename self.subPkgDir with a '.tmp' suffix
# 3. Rename self.subPkgTempDir to become self.subPkgDir
# 4. Remove self.subPkgDir + '.tmp'
# System is always stable, excelt for the split-second between 2 and 3.
# Read list of new sub-packages
newInfoDict=self._readInfoFiles(os.path.join(self.subPkgTempDir, "info"))
self._log("add-sub-pkg").info("add(): Found new sub-pkgs %s", newInfoDict)
newSubPkgs=newInfoDict.keys()
# Sanity-check the new info: Make sure new sub-pkgs have all the data we need in their .info files
for subPkg in newSubPkgs:
info=newInfoDict[subPkg]
if not 'dir' in info:
self._log("add-bad-info-dir").error("add(): sub-pkg %s info does not have 'dir'", subPkg)
return None
infoDirPath=os.path.join(self.subPkgTempDir, info['dir'])
if not os.path.isdir(infoDirPath):
self._log("add-missing-dir").error("add(): sub-pkg %s info has 'dir'->%s, does not exist",
subPkg, infoDirPath)
return None
if not 'name' in info:
self._log("add-bad-info-name").error("add(): sub-pkg %s info does not have 'name'", subPkg)
return None
if info['name'] != subPkg:
self._log("add-bad-name").error("add(): sub-pkg %s info has 'name'=%s, should be identical",
subPkg, info['name'])
return None
# Step 1: Copy all old sub-packages, which are not found in the new dir, to the new dir
for subPkg in oldSubPkgs:
if subPkg not in newSubPkgs:
oldInfoFile=self._getInfoFileName(self.subPkgDir, subPkg)
newInfoFile=self._getInfoFileName(self.subPkgTempDir, subPkg)
oldSubPkgDir=os.path.join(self.subPkgDir, oldInfoDict[subPkg]['dir'])
newSubPkgDir=os.path.join(self.subPkgTempDir, oldInfoDict[subPkg]['dir'])
if os.path.exists(newSubPkgDir):
self._log("add-bad-name").error("add(): old sub-pkg %s not found in new sub-packages, but %s exists",
subPkg, newSubPkgDir)
return None
self.utils.runCommandRaiseIfFail("cp -vpf %s %s" % (oldInfoFile, newInfoFile))
self.utils.runCommandRaiseIfFail("cp -vrpf %s %s" % (oldSubPkgDir, newSubPkgDir))
# Steps 2-4: Copy all old sub-packages, which are not found in the new dir, to the new dir
tmpOldName = self.subPkgDir+'.tmp'
self.utils.runCommandRaiseIfFail("mv -vf %s %s" % (self.subPkgDir, tmpOldName))
self.utils.runCommandRaiseIfFail("mv -vf %s %s" % (self.subPkgTempDir, self.subPkgDir))
self.utils.runCommandRaiseIfFail("rm -vrf %s" % (tmpOldName))
finally:
# Remove the install-temp directory, no sense in leaving it here
self._cleanTempDir()
# oldInfoDict.keys() holds the list of subPkgs before the install. Let's find out what was added.
newInfoDict=self._readInfoFiles(self.infoDir)
# Sorting makes it easy on unit tests :-)
addedSubPkgs=sorted(list(set(newInfoDict.keys())-set(oldInfoDict.keys())))
return (sorted(newSubPkgs), addedSubPkgs)
def removeSubPackages (self, subPackages):
"""
Removes a set of existing sub-pkgs
"""
self._log("remove-sub-packages").info("removeSubPackages() called, subPackages=%s", subPackages)
infoDict=self._readInfoFiles(self.infoDir)
for subPkg in subPackages:
if not subPkg in infoDict:
self._log("remove-sub-packages").error("removeSubPackages(): sub-package %s not found", subPkg)
continue
subPkgDir=os.path.join(self.subPkgDir, infoDict[subPkg]['dir'])
self.utils.runCommandRaiseIfFail("rm -rf %s" % subPkgDir)
infoFileName=self._getInfoFileName(self.subPkgDir, subPkg)
self.utils.runCommandRaiseIfFail("rm %s" % infoFileName)
def addReposToYum (self, yum, repoFilter=None):
self._log("add-repos-to-yum").info("addReposToYum() called, repoFilter=%s", repoFilter)
infoDict=self._readInfoFiles(self.infoDir)
for info in infoDict.keys():
infoData=infoDict[info]
if infoData['type'] == 'repository':
repoDir=os.path.join(self.subPkgDir, infoData['dir'])
repoName=infoData['name']
if (repoFilter is None) or (repoName.find(repoFilter) != -1):
self._log("add-repos-to-yum").info("addReposToYum(): Adding repoDir=%s", repoDir)
yum.addDiskRepo(repoName, repoDir)
def getSubPkgs (self):
return self._readInfoFiles(self.infoDir)
def getSubPkgsDir (self):
return self.subPkgDir
def _getFilePathFromRoot (self, r2pmFile):
if not self.useFakeChrootMode:
return r2pmFile
if r2pmFile.startswith(self.rootDir):
return r2pmFile[len(self.rootDir):]
raise ValueError("File %s not in root dir %s" % (r2pmFile, self.rootDir))
def _readInfoFiles (self, fromDir):
"""
Returns a dict of all info files found in fromDir.
For each file <fromDir>/X.info: X is key, value is a dict with the content of the info file
"""
self._raiseIfNotAbsPath(fromDir, "fromDir")
# No directory ? No sub-pkgs !
if not os.path.exists(fromDir):
return {}
files=os.listdir(fromDir)
ret={}
for file_ in files:
mo=re.match("^(.*)\.info$",file_)
if mo != None:
subPkg=mo.group(1)
ret[subPkg]=self._readInfoFile(os.path.join(fromDir, file_))
return ret
def _readInfoFile (self, infoFileName):
f=open(infoFileName,"r")
text=f.read()
f.close()
self._log("read-info-file").info("_readInfoFile(): Info file %s has text %s", infoFileName, text)
return json.loads(text)
def _getInfoFileName (self, root, subPkg):
return os.path.join(root, "info", subPkg+".info")
def _cleanTempDir (self):
self.utils.runCommandRaiseIfFail("rm -rf "+self.subPkgTempDir)
def _raiseIfNotAbsPath (self, pathToCheck, name):
if not pathToCheck.startswith('/'):
raise ValueError("%s must start with '/', value given is '%s'" % (name, pathToCheck))
class PilotServices(object):
"""
######################################################################################################
# NOTE: ALL public funcs are part of the pilot API, changes must be considered carefully !
# Better not to change anything, only add
######################################################################################################
"""
def __init__(self, logger, stateManager, installManager, pilotVersion,
pilotDir, logDir, workDir, apiDir, startupScriptsDir,
rpmActiveRfs, yumActiveRfs, yumConfForActiveRfs,
yumConfForNewVersionOnly, prevRfsDir, sdUtils, bootUtils):
self._log = logger.createLogger("sys-install", "pilot-services")
self._utils = Utils(self._log)
self._stateManager = stateManager
self._installManager = installManager
self._pilotVersion = pilotVersion
self._pilotDir = pilotDir
self._logDir = logDir
self._workDir = workDir
self._apiDir = apiDir
self._startupScriptsDir = startupScriptsDir
self._rpmActiveRfs = rpmActiveRfs
self._yumActiveRfs = yumActiveRfs
self._yumConfForActiveRfs = yumConfForActiveRfs
self._yumConfForNewVersionOnly = yumConfForNewVersionOnly
self._debugNoInstallMode = False
self._otherPilot = None
self._pilotHints = None
self._prevRfsDir = prevRfsDir
self._sdUtils = sdUtils
self._bootUtils = bootUtils
def debugSetNoInstallMode(self):
self._debugNoInstallMode = True
def setOtherPilot(self, otherPilot):
"""NOT FOR USE BY PILOT !!!"""
self._otherPilot = otherPilot
def setPilotHints(self, hints):
"""NOT FOR USE BY PILOT !!!"""
self._pilotHints = hints
def logInfo(self, msg):
"""Logs an info message"""
self._log("pilot-info").info(msg)
def logWarning(self, msg):
"""Logs a warning message"""
self._log("pilot-warning").warning(msg)
def logError(self, msg):
"""Logs an error message"""
self._log("pilot-error").error(msg)
def raiseException(self, msg):
"""Raises an exception to abort current pilot operation with an error"""
self._log("raise-exception").info("raiseException() called, msg = %s",
msg)
raise InstallException(msg)
def getPilotHints(self):
"""Returns the pilot hints, None if none"""
return self._pilotHints
def getStateManager(self):
return self._stateManager
def getPilotVersion(self):
"""Returns Version of pilot being serviced"""
self._log("get-pilot-version").info(
"getPilotVersion() called, returning %s", self._pilotVersion)
return self._pilotVersion
def getPilotDir(self):
"""Returns absolute path to pilot directory, for use by pilot to access other files"""
self._log("get-pilot-dir").info("getPilotDir() called, returning %s",
self._pilotDir)
return self._pilotDir
def getOtherPilot(self):
"""Returns the other pilot. Use this only during prepare() !!!"""
return self._otherPilot
def getSubPackages(self):
"""
Returns a dict of all sub-packages installed.
For each sub-package: sub-package name is key, value is a dict with the content of the info file of the sub-package
"""
self._log("get-sub-packages").info("getSubPackages() called")
return self._installManager.getSubPkgManager().getSubPkgs()
def getSubPkgsDir(self):
"""Returns the directory in which all sub-packages are installed"""
self._log("get-sub-packages-dir").info("getSubPkgsDir() called")
return self._installManager.getSubPkgManager().getSubPkgsDir()
def getBootDevPath(self):
"""Returns the path to the boot device"""
bd = self._installManager.getBootDevPath()
self._log("get-boot-dev-path").info(
"getBootDevPath() called, returning %s", bd)
return bd
def getVitalDir(self):
"""Returns the path to the vital dir"""
vd = self._installManager.getVitalDir()
self._log("get-vital-dir").info("getVitalDir() called, returning %s",
vd)
return vd
def getGrubConf(self):
"""Returns the name of the grub.conf file"""
gc = self._installManager.getGrubConf()
self._log("get-grub-conf").info("getGrubConf() called, returning %s",
gc)
return gc
def getAppDir(self):
"""Returns absolute path to app directory"""
self._log("get-app-dir").info("getAppDir() called")
appDir = self._installManager.getAppDir()
self._log("get-app-dir").info("getAppDir() returning %s", appDir)
return appDir
def getAppDirReal(self):
"""Returns absolute path to real app directory"""
self._log("get-app-dir-real").info("getAppDirReal() called")
appDirReal = self._installManager.getAppDirReal()
self._log("get-app-dir-real").info("getAppDirReal() returning %s",
appDirReal)
return appDirReal
def getRpmsDir(self):
"""Returns absolute path to rpms directory"""
self._log("get-app-dir").info("getRpmsDir() called")
rpmsDir = self._installManager.getRpmsDir()
self._log("get-app-dir").info("getRpmsDir() returning %s", rpmsDir)
return rpmsDir
def getLogDir(self):
"""Returns absolute path to log directory, for use by pilot to create log files"""
self._log("get-log-dir").info("getLogDir() called, returning %s",
self._logDir)
return self._logDir
def getWorkDir(self):
"""Returns absolute path to work directory, for use by pilot to create scripts and such"""
self._log("get-work-dir").info("getWorkDir() called, returning %s",
self._workDir)
return self._workDir
def getApiDir(self):
"""Returns absolute path to api directory, for use by pilot to store information available to other modules/processes"""
self._log("get-api-dir").info("getApiDir() called, returning %s",
self._apiDir)
return self._apiDir
def getStartupScriptsDir(self):
"""
Returns absolute path to startup scripts directory, for use by pilot to create scripts executed
by oscar on the next system start
"""
self._log("get-startup-scripts-dir").info(
"getStartupScriptsDir() called, returning %s",
self._startupScriptsDir)
return self._startupScriptsDir
def getPrevRfsDir(self):
self._log("get-prev-rfs-dir").info(
"getPrevRfsDir() called, returning %s", self._prevRfsDir)
return self._prevRfsDir
def getSdUtils(self):
self._log("get-sd-utils").info(
"getSdUtils() called, returning object %s", self._sdUtils)
return self._sdUtils
def getBootUtils(self):
self._log("get-boot-utils").info(
"getBootUtils() called, returning object %s", self._bootUtils)
return self._bootUtils
def getPreparedPackageFilePath(self):
fp = self._installManager.getPreparedPackageFilePath()
self._log("get-prepared-package-file-path").info(
"getPreparedPackageFilePath() called, returning object %s", fp)
return fp
def rpmListRequiresActiveRfs(self, package):
"""
Gets list of required capabilities of a package installed in the active RFS.
package: Package name
Returns List of tuples: (capName, capCondition, capVersion)
"""
self._log("rpm-list-required-active-rfs-called").info(
"rpmListRequiredActiveRfs() called, package=%s", package)
ret = self._rpmActiveRfs.doListRequires(package)
self._log("rpm-list-required-active-rfs-returning").info(
"rpmListRequiredActiveRfs() returning %s", ret)
return ret
def yumTestInstallActiveRfs(self, packages):
"""
Test-Install a set of packages on the active RFS.
packages: A space-separated string of package names
Returns True if OK, False if not
"""
self._log("yum-test-install-active-rfs-called").info(
"yumTestInstallActiveRfs() called, packages=%s", packages)
ret = self._yumActiveRfs.doTestInstall(packages)
self._log("yum-test-install-active-rfs-returning").info(
"yumTestInstallActiveRfs() returning %s", ret)
return ret
def yumInstallActiveRfs(self, packages):
"""
Installs a set of packages on the active RFS.
packages: A space-separated string of package names
"""
self._log("yum-install-active-rfs-called").info(
"yumInstallActiveRfs() called, packages=%s", packages)
if not self._debugNoInstallMode:
self._yumActiveRfs.doInstall(packages)
self._log("yum-install-active-rfs-done").info(
"yumInstallActiveRfs() done")
def getYumConfigFileActiveRfs(self):
"""
Gets the name of a yum config file, configured for all current repositories (found in sub-packages)
"""
self._log("get-yum-config-file-active-rfs-called").info(
"getYumConfigFileActiveRfs() called, returning %s",
self._yumConfForActiveRfs)
return self._yumConfForActiveRfs
def getYumConfigFileNewVersionOnly(self):
"""
Gets the name of a yum config file, configured for one repository: The one that came with the currently installed version
"""
self._log("get-yum-config-file-new-version-only-called").info(
"getYumConfigFileNewVersionOnly() called, returning %s",
self._yumConfForNewVersionOnly)
return self._yumConfForNewVersionOnly
def getAvailableSize(self, path_):
"""
Returns available size in bytes on a given path
"""
self._log("get-avail-size").info("getAvailableSize() called, path=%s",
path_)
return self._utils.getAvailableSize(path_)
def hook_runYumActiveRfs(self, command):
"""Emergency use only. Allows pilot to run any yum command"""
self._log("hook-run-yum-active-rfs-called").info(
"hook_runYumActiveRfs() called, command=%s", command)
(rc, outText, errText) = self._rpmActiveRfs._runRpm(command)
self._log("hook-run-yum-active-rfs-returning").info(
"hook_runYumActiveRfs() returning rc=%s, outText=%s, errText=%s",
rc, outText, errText)
return (rc, outText, errText)
def hook_runRpmActiveRfs(self, command):
"""Emergency use only. Allows pilot to run any rpm command"""
self._log("hook-run-rpm-active-rfs-called").info(
"hook_runRpmActiveRfs() called, command=%s", command)
(rc, outText, errText) = self._yumActiveRfs._runYum(command)
self._log("hook-run-rpm-active-rfs-returning").info(
"hook_runRpmActiveRfs() returning rc=%s, outText=%s, errText=%s",
rc, outText, errText)
return (rc, outText, errText)
def hook_runCommand(self, command):
"""
Allows pilot to run any command.
Returns (rc, outText, errText)
"""
self._log("hook-run-command-called").info(
"hook_runCommand() called, command=%s", command)
(rc, outText, errText) = self._utils.runCommand(command)
self._log("hook-run-command-returning").info(
"hook_runCommand() returning rc=%s", rc)
return (rc, outText, errText)
def __init__ (self, logger):
self._log=logger.createLogger("sys-install","rpm-rpm")
self.utils=Utils(self._log)
self.rootDir=None
self.useFakeChrootMode=False
self._updateCommand()
class Rpm(object):
def __init__ (self, logger):
self._log=logger.createLogger("sys-install","rpm-rpm")
self.utils=Utils(self._log)
self.rootDir=None
self.useFakeChrootMode=False
self._updateCommand()
def setRoot (self, rootDir, useFakeChrootMode=False):
self._raiseIfNotAbsPath(rootDir, "rootDir")
self.rootDir=rootDir
self.useFakeChrootMode=useFakeChrootMode
self._updateCommand()
def doImport (self, keyPath):
"""
Imports a public key (from an armoured ascii file) into the RPM db
"""
self._log("do-import-called").info("doImport(%s) called", keyPath)
self._raiseIfNotAbsPath(keyPath, "keyPath")
self._runRpmRaiseIfFail("--import "+keyPath)
def doValidateRpm (self, rpmPath):
"""
Checks that a given RPM file is properly signed
Returns True if file is properly signed
"""
self._log("do-validate-rpm-called").info("doValidateRpm(%s) called", rpmPath)
self._raiseIfNotAbsPath(rpmPath, "rpmPath")
(rc,outText,_)=self._runRpm("--checksig "+rpmPath)
if rc != 0:
self._log("do-validate-rpm-called").warning("doValidateRpm(%s): rpm returned %s", rpmPath, rc)
return False
if re.search("rsa ", outText) == None:
self._log("do-validate-rpm-called").warning("doValidateRpm(%s): rpm output is '%s', no 'rsa ' found", rpmPath, outText)
return False
if re.search(" pgp ", outText) == None:
self._log("do-validate-rpm-called").warning("doValidateRpm(%s): rpm output is '%s', no ' gpg ' found", rpmPath, outText)
return False
return True
def doVerifyAllInstalled (self):
"""
Verifies all install packages, comparing files on disk against md5sums remembered from package.
Returns list of strings with problems. If no problems, returns empty string
"""
self._log("do-verify-all-installed-called").info("doVerifyAllInstalled() called")
cmd=" --verify --all"
# --nomode is required due to the fakeroot thing used for unit testing: a file with -r--r--r-- permissions
# inside the RPM is generated as a -rw-r--r-- file when installed
if self.useFakeChrootMode:
cmd=cmd+" --nomode"
(rc, outText, _)=self._runRpm(cmd)
if rc==0:
return []
return filter(lambda x: x.strip() != "", outText.split('\n'))
def isInstalled (self, package):
self._log("is-installed-called").info("isInstalled(%s) called", package)
(rc,outText,errText)=self._runRpm("-q "+package)
if (rc==1) and (outText.find("is not installed") != -1):
return False
if rc==0:
return True
raise RuntimeError("RPM command 'rpm -q %s' failed" % package)
def doInstall (self, rpmPath):
self._log("do-install-called").info("doInstall(%s) called", rpmPath)
self._raiseIfNotAbsPath(rpmPath, "rpmPath")
self._runRpmRaiseIfFail("--install "+rpmPath)
def doRemove (self, package):
self._log("do-remove-called").info("doRemove(%s) called", package)
self._runRpmRaiseIfFail("--erase "+package)
def doListFiles (self, rpmPath):
"""Returns a list of files in an RPM file"""
self._log("do-list-files-called").info("doListFiles(%s) called", rpmPath)
self._raiseIfNotAbsPath(rpmPath, "rpmPath")
(outText,errText)=self._runRpmRaiseIfFail("-qpl "+rpmPath)
files=outText.split()
self._log("do-list-files-returning").info("doListFiles(%s) returning '%s'", rpmPath, files)
return sorted(files)
def doListRequires (self, package):
"""
Returns a list of capabilities that a package required. Package should be installed.
If package is not installed, returns an empty list
List is made of (capName, capCondition, capVersion) tuples:
capName is name of capability
capCondition is the '=', '<=' string, or None if no condition was specified
capVersion is the required version, or None if no condition was specified
"""
self._log("do-list-requires-called").info("doListRequires(%s) called", package)
(rc, outText, errText)=self._runRpm("-q --requires "+package)
if (rc==1) and (outText.find("is not installed") != -1):
return []
ret=self._parseCapabilities(outText)
self._log("do-list-requires-returning").info("doListRequires(%s) returning '%s'", package, ret)
return ret
def doListRequiresFile (self, rpmFile):
"""
Returns a list of capabilities that a package required. Package is an RPM file.
List is in the same format as returned by doListRequires()
"""
self._log("do-list-requires-file-called").info("doListRequiresFile(%s) called", rpmFile)
(outText, errText)=self._runRpmRaiseIfFail("-qp --requires "+rpmFile)
ret=self._parseCapabilities(outText)
self._log("do-list-requires-file-returning").info("doListRequiresFile(%s) returning '%s'", rpmFile, ret)
return ret
def doListProvides (self, package):
"""
Returns a list of capabilities that a package provides. Package should be installed.
If package is not installed, returns an empty list
List is in the same format as returned by doListRequires()
"""
self._log("do-list-provides-called").info("doListProvides(%s) called", package)
(rc, outText, errText)=self._runRpm("-q --provides "+package)
if (rc==1) and (outText.find("is not installed") != -1):
return []
ret=self._parseCapabilities(outText)
self._log("do-list-provides-returning").info("doListProvides(%s) returning '%s'", package, ret)
return ret
def doListProvidesFile (self, rpmFile):
"""
Returns a list of capabilities that a package required. Package is an RPM file.
List is in the same format as returned by doListRequires()
"""
self._log("do-list-provides-file-called").info("doListProvidesFile(%s) called", rpmFile)
(outText, errText)=self._runRpmRaiseIfFail("-qp --provides "+rpmFile)
ret=self._parseCapabilities(outText)
self._log("do-list-provides-file-returning").info("doListProvidesFile(%s) returning '%s'", rpmFile, ret)
return ret
def doGetInstalledPackages (self):
"""Returns a list of installed packages"""
self._log("do-get-installed-packages-called").info("doGetInstalledPackages() called")
(outText,errText)=self._runRpmRaiseIfFail("-qa")
packages=outText.split()
self._log("do-get-installed-packages-returning").info("doGetInstalledPackages() returning '%s'", packages)
return sorted(packages)
def _parseCapabilities(self, outText):
lines=outText.strip().split('\n')
ret=[]
for line in lines:
# Sometimes RPM files the DB locked, as a result of a killed -9 yum command, and as a result,
# it prints such lines as part of the output. We ignre these lines
if line.startswith("Freeing read locks"):
continue
tokens=line.strip().split()
if len(tokens)==3:
ret.append((tokens[0], tokens[1], tokens[2]))
elif len(tokens)==1:
ret.append((tokens[0], None, None))
else:
self._log("parse-capabilities-error").error("_parseCapabilities() got a line '%s', un-parsable", line)
raise RuntimeError("_parseCapabilities() got a line '%s', un-parsable" % line)
return ret
def _raiseIfNotAbsPath (self, pathToCheck, name):
if not pathToCheck.startswith('/'):
raise ValueError("%s must start with '/', value given is '%s'" % (name, pathToCheck))
def _runRpm (self, command):
"""
Returns a tuple (rc, outText,errText)
NOTE: This function is used by pilot services to allow pilot to run the rpm command
"""
cmd=self.command+" "+command
(rc,outText,errText)=self.utils.runCommand(cmd)
return (rc,outText,errText)
def _runRpmRaiseIfFail (self, command):
"""Returns a tuple (outText,errText)"""
(rc,outText,errText)=self._runRpm(command)
if rc != 0:
raise RuntimeError("RPM command '%s' failed" % command)
return (outText,errText)
def _updateCommand (self):
self.command="rpm"
if self.useFakeChrootMode:
self.command="fakeroot fakechroot /usr/sbin/chroot "+self.rootDir+" "+self.command
elif (self.rootDir != None) and (self.rootDir != '/'):
self.command = self.command+" --root "+self.rootDir
def __init__(self, logger):
self._log = logger.createLogger("sys-install", "rpm-rpm")
self.utils = Utils(self._log)
self.rootDir = None
self.useFakeChrootMode = False
self._updateCommand()
class Rpm(object):
def __init__(self, logger):
self._log = logger.createLogger("sys-install", "rpm-rpm")
self.utils = Utils(self._log)
self.rootDir = None
self.useFakeChrootMode = False
self._updateCommand()
def setRoot(self, rootDir, useFakeChrootMode=False):
self._raiseIfNotAbsPath(rootDir, "rootDir")
self.rootDir = rootDir
self.useFakeChrootMode = useFakeChrootMode
self._updateCommand()
def doImport(self, keyPath):
"""
Imports a public key (from an armoured ascii file) into the RPM db
"""
self._log("do-import-called").info("doImport(%s) called", keyPath)
self._raiseIfNotAbsPath(keyPath, "keyPath")
self._runRpmRaiseIfFail("--import " + keyPath)
def doValidateRpm(self, rpmPath):
"""
Checks that a given RPM file is properly signed
Returns True if file is properly signed
"""
self._log("do-validate-rpm-called").info("doValidateRpm(%s) called",
rpmPath)
self._raiseIfNotAbsPath(rpmPath, "rpmPath")
(rc, outText, _) = self._runRpm("--checksig " + rpmPath)
if rc != 0:
self._log("do-validate-rpm-called").warning(
"doValidateRpm(%s): rpm returned %s", rpmPath, rc)
return False
if re.search("rsa ", outText) == None:
self._log("do-validate-rpm-called").warning(
"doValidateRpm(%s): rpm output is '%s', no 'rsa ' found",
rpmPath, outText)
return False
if re.search(" pgp ", outText) == None:
self._log("do-validate-rpm-called").warning(
"doValidateRpm(%s): rpm output is '%s', no ' gpg ' found",
rpmPath, outText)
return False
return True
def doVerifyAllInstalled(self):
"""
Verifies all install packages, comparing files on disk against md5sums remembered from package.
Returns list of strings with problems. If no problems, returns empty string
"""
self._log("do-verify-all-installed-called").info(
"doVerifyAllInstalled() called")
cmd = " --verify --all"
# --nomode is required due to the fakeroot thing used for unit testing: a file with -r--r--r-- permissions
# inside the RPM is generated as a -rw-r--r-- file when installed
if self.useFakeChrootMode:
cmd = cmd + " --nomode"
(rc, outText, _) = self._runRpm(cmd)
if rc == 0:
return []
return filter(lambda x: x.strip() != "", outText.split('\n'))
def isInstalled(self, package):
self._log("is-installed-called").info("isInstalled(%s) called",
package)
(rc, outText, errText) = self._runRpm("-q " + package)
if (rc == 1) and (outText.find("is not installed") != -1):
return False
if rc == 0:
return True
raise RuntimeError("RPM command 'rpm -q %s' failed" % package)
def doInstall(self, rpmPath):
self._log("do-install-called").info("doInstall(%s) called", rpmPath)
self._raiseIfNotAbsPath(rpmPath, "rpmPath")
self._runRpmRaiseIfFail("--install " + rpmPath)
def doRemove(self, package):
self._log("do-remove-called").info("doRemove(%s) called", package)
self._runRpmRaiseIfFail("--erase " + package)
def doListFiles(self, rpmPath):
"""Returns a list of files in an RPM file"""
self._log("do-list-files-called").info("doListFiles(%s) called",
rpmPath)
self._raiseIfNotAbsPath(rpmPath, "rpmPath")
(outText, errText) = self._runRpmRaiseIfFail("-qpl " + rpmPath)
files = outText.split()
self._log("do-list-files-returning").info(
"doListFiles(%s) returning '%s'", rpmPath, files)
return sorted(files)
def doListRequires(self, package):
"""
Returns a list of capabilities that a package required. Package should be installed.
If package is not installed, returns an empty list
List is made of (capName, capCondition, capVersion) tuples:
capName is name of capability
capCondition is the '=', '<=' string, or None if no condition was specified
capVersion is the required version, or None if no condition was specified
"""
self._log("do-list-requires-called").info("doListRequires(%s) called",
package)
(rc, outText, errText) = self._runRpm("-q --requires " + package)
if (rc == 1) and (outText.find("is not installed") != -1):
return []
ret = self._parseCapabilities(outText)
self._log("do-list-requires-returning").info(
"doListRequires(%s) returning '%s'", package, ret)
return ret
def doListRequiresFile(self, rpmFile):
"""
Returns a list of capabilities that a package required. Package is an RPM file.
List is in the same format as returned by doListRequires()
"""
self._log("do-list-requires-file-called").info(
"doListRequiresFile(%s) called", rpmFile)
(outText,
errText) = self._runRpmRaiseIfFail("-qp --requires " + rpmFile)
ret = self._parseCapabilities(outText)
self._log("do-list-requires-file-returning").info(
"doListRequiresFile(%s) returning '%s'", rpmFile, ret)
return ret
def doListProvides(self, package):
"""
Returns a list of capabilities that a package provides. Package should be installed.
If package is not installed, returns an empty list
List is in the same format as returned by doListRequires()
"""
self._log("do-list-provides-called").info("doListProvides(%s) called",
package)
(rc, outText, errText) = self._runRpm("-q --provides " + package)
if (rc == 1) and (outText.find("is not installed") != -1):
return []
ret = self._parseCapabilities(outText)
self._log("do-list-provides-returning").info(
"doListProvides(%s) returning '%s'", package, ret)
return ret
def doListProvidesFile(self, rpmFile):
"""
Returns a list of capabilities that a package required. Package is an RPM file.
List is in the same format as returned by doListRequires()
"""
self._log("do-list-provides-file-called").info(
"doListProvidesFile(%s) called", rpmFile)
(outText,
errText) = self._runRpmRaiseIfFail("-qp --provides " + rpmFile)
ret = self._parseCapabilities(outText)
self._log("do-list-provides-file-returning").info(
"doListProvidesFile(%s) returning '%s'", rpmFile, ret)
return ret
def doGetInstalledPackages(self):
"""Returns a list of installed packages"""
self._log("do-get-installed-packages-called").info(
"doGetInstalledPackages() called")
(outText, errText) = self._runRpmRaiseIfFail("-qa")
packages = outText.split()
self._log("do-get-installed-packages-returning").info(
"doGetInstalledPackages() returning '%s'", packages)
return sorted(packages)
def _parseCapabilities(self, outText):
lines = outText.strip().split('\n')
ret = []
for line in lines:
# Sometimes RPM files the DB locked, as a result of a killed -9 yum command, and as a result,
# it prints such lines as part of the output. We ignre these lines
if line.startswith("Freeing read locks"):
continue
tokens = line.strip().split()
if len(tokens) == 3:
ret.append((tokens[0], tokens[1], tokens[2]))
elif len(tokens) == 1:
ret.append((tokens[0], None, None))
else:
self._log("parse-capabilities-error").error(
"_parseCapabilities() got a line '%s', un-parsable", line)
raise RuntimeError(
"_parseCapabilities() got a line '%s', un-parsable" % line)
return ret
def _raiseIfNotAbsPath(self, pathToCheck, name):
if not pathToCheck.startswith('/'):
raise ValueError("%s must start with '/', value given is '%s'" %
(name, pathToCheck))
def _runRpm(self, command):
"""
Returns a tuple (rc, outText,errText)
NOTE: This function is used by pilot services to allow pilot to run the rpm command
"""
cmd = self.command + " " + command
(rc, outText, errText) = self.utils.runCommand(cmd)
return (rc, outText, errText)
def _runRpmRaiseIfFail(self, command):
"""Returns a tuple (outText,errText)"""
(rc, outText, errText) = self._runRpm(command)
if rc != 0:
raise RuntimeError("RPM command '%s' failed" % command)
return (outText, errText)
def _updateCommand(self):
self.command = "rpm"
if self.useFakeChrootMode:
self.command = "fakeroot fakechroot /usr/sbin/chroot " + self.rootDir + " " + self.command
elif (self.rootDir != None) and (self.rootDir != '/'):
self.command = self.command + " --root " + self.rootDir
class PilotServices(object):
"""
######################################################################################################
# NOTE: ALL public funcs are part of the pilot API, changes must be considered carefully !
# Better not to change anything, only add
######################################################################################################
"""
def __init__ (self, logger, stateManager, installManager, pilotVersion, pilotDir, logDir, workDir, apiDir, startupScriptsDir,
rpmActiveRfs, yumActiveRfs, yumConfForActiveRfs, yumConfForNewVersionOnly, prevRfsDir, sdUtils, bootUtils):
self._log = logger.createLogger("sys-install","pilot-services")
self._utils = Utils(self._log)
self._stateManager = stateManager
self._installManager = installManager
self._pilotVersion = pilotVersion
self._pilotDir = pilotDir
self._logDir = logDir
self._workDir = workDir
self._apiDir = apiDir
self._startupScriptsDir = startupScriptsDir
self._rpmActiveRfs = rpmActiveRfs
self._yumActiveRfs = yumActiveRfs
self._yumConfForActiveRfs = yumConfForActiveRfs
self._yumConfForNewVersionOnly = yumConfForNewVersionOnly
self._debugNoInstallMode = False
self._otherPilot = None
self._pilotHints = None
self._prevRfsDir = prevRfsDir
self._sdUtils = sdUtils
self._bootUtils = bootUtils
def debugSetNoInstallMode (self):
self._debugNoInstallMode=True
def setOtherPilot (self, otherPilot):
"""NOT FOR USE BY PILOT !!!"""
self._otherPilot = otherPilot
def setPilotHints (self, hints):
"""NOT FOR USE BY PILOT !!!"""
self._pilotHints = hints
def logInfo (self, msg):
"""Logs an info message"""
self._log("pilot-info").info(msg)
def logWarning (self, msg):
"""Logs a warning message"""
self._log("pilot-warning").warning(msg)
def logError (self, msg):
"""Logs an error message"""
self._log("pilot-error").error(msg)
def raiseException (self, msg):
"""Raises an exception to abort current pilot operation with an error"""
self._log("raise-exception").info("raiseException() called, msg = %s", msg)
raise InstallException(msg)
def getPilotHints (self):
"""Returns the pilot hints, None if none"""
return self._pilotHints
def getStateManager (self):
return self._stateManager
def getPilotVersion (self):
"""Returns Version of pilot being serviced"""
self._log("get-pilot-version").info("getPilotVersion() called, returning %s", self._pilotVersion)
return self._pilotVersion
def getPilotDir (self):
"""Returns absolute path to pilot directory, for use by pilot to access other files"""
self._log("get-pilot-dir").info("getPilotDir() called, returning %s", self._pilotDir)
return self._pilotDir
def getOtherPilot (self):
"""Returns the other pilot. Use this only during prepare() !!!"""
return self._otherPilot
def getSubPackages (self):
"""
Returns a dict of all sub-packages installed.
For each sub-package: sub-package name is key, value is a dict with the content of the info file of the sub-package
"""
self._log("get-sub-packages").info("getSubPackages() called")
return self._installManager.getSubPkgManager().getSubPkgs()
def getSubPkgsDir (self):
"""Returns the directory in which all sub-packages are installed"""
self._log("get-sub-packages-dir").info("getSubPkgsDir() called")
return self._installManager.getSubPkgManager().getSubPkgsDir()
def getBootDevPath (self):
"""Returns the path to the boot device"""
bd=self._installManager.getBootDevPath()
self._log("get-boot-dev-path").info("getBootDevPath() called, returning %s", bd)
return bd
def getVitalDir (self):
"""Returns the path to the vital dir"""
vd=self._installManager.getVitalDir()
self._log("get-vital-dir").info("getVitalDir() called, returning %s", vd)
return vd
def getGrubConf (self):
"""Returns the name of the grub.conf file"""
gc=self._installManager.getGrubConf()
self._log("get-grub-conf").info("getGrubConf() called, returning %s", gc)
return gc
def getAppDir (self):
"""Returns absolute path to app directory"""
self._log("get-app-dir").info("getAppDir() called")
appDir=self._installManager.getAppDir()
self._log("get-app-dir").info("getAppDir() returning %s", appDir)
return appDir
def getAppDirReal (self):
"""Returns absolute path to real app directory"""
self._log("get-app-dir-real").info("getAppDirReal() called")
appDirReal=self._installManager.getAppDirReal()
self._log("get-app-dir-real").info("getAppDirReal() returning %s", appDirReal)
return appDirReal
def getRpmsDir (self):
"""Returns absolute path to rpms directory"""
self._log("get-app-dir").info("getRpmsDir() called")
rpmsDir=self._installManager.getRpmsDir()
self._log("get-app-dir").info("getRpmsDir() returning %s", rpmsDir)
return rpmsDir
def getLogDir (self):
"""Returns absolute path to log directory, for use by pilot to create log files"""
self._log("get-log-dir").info("getLogDir() called, returning %s", self._logDir)
return self._logDir
def getWorkDir (self):
"""Returns absolute path to work directory, for use by pilot to create scripts and such"""
self._log("get-work-dir").info("getWorkDir() called, returning %s", self._workDir)
return self._workDir
def getApiDir (self):
"""Returns absolute path to api directory, for use by pilot to store information available to other modules/processes"""
self._log("get-api-dir").info("getApiDir() called, returning %s", self._apiDir)
return self._apiDir
def getStartupScriptsDir (self):
"""
Returns absolute path to startup scripts directory, for use by pilot to create scripts executed
by oscar on the next system start
"""
self._log("get-startup-scripts-dir").info("getStartupScriptsDir() called, returning %s", self._startupScriptsDir)
return self._startupScriptsDir
def getPrevRfsDir (self):
self._log("get-prev-rfs-dir").info("getPrevRfsDir() called, returning %s", self._prevRfsDir)
return self._prevRfsDir
def getSdUtils (self):
self._log("get-sd-utils").info("getSdUtils() called, returning object %s", self._sdUtils)
return self._sdUtils
def getBootUtils (self):
self._log("get-boot-utils").info("getBootUtils() called, returning object %s", self._bootUtils)
return self._bootUtils
def getPreparedPackageFilePath (self):
fp = self._installManager.getPreparedPackageFilePath()
self._log("get-prepared-package-file-path").info("getPreparedPackageFilePath() called, returning object %s", fp)
return fp
def rpmListRequiresActiveRfs (self, package):
"""
Gets list of required capabilities of a package installed in the active RFS.
package: Package name
Returns List of tuples: (capName, capCondition, capVersion)
"""
self._log("rpm-list-required-active-rfs-called").info("rpmListRequiredActiveRfs() called, package=%s", package)
ret = self._rpmActiveRfs.doListRequires(package)
self._log("rpm-list-required-active-rfs-returning").info("rpmListRequiredActiveRfs() returning %s", ret)
return ret
def yumTestInstallActiveRfs (self, packages):
"""
Test-Install a set of packages on the active RFS.
packages: A space-separated string of package names
Returns True if OK, False if not
"""
self._log("yum-test-install-active-rfs-called").info("yumTestInstallActiveRfs() called, packages=%s", packages)
ret = self._yumActiveRfs.doTestInstall(packages)
self._log("yum-test-install-active-rfs-returning").info("yumTestInstallActiveRfs() returning %s", ret)
return ret
def yumInstallActiveRfs (self, packages):
"""
Installs a set of packages on the active RFS.
packages: A space-separated string of package names
"""
self._log("yum-install-active-rfs-called").info("yumInstallActiveRfs() called, packages=%s", packages)
if not self._debugNoInstallMode:
self._yumActiveRfs.doInstall(packages)
self._log("yum-install-active-rfs-done").info("yumInstallActiveRfs() done")
def getYumConfigFileActiveRfs (self):
"""
Gets the name of a yum config file, configured for all current repositories (found in sub-packages)
"""
self._log("get-yum-config-file-active-rfs-called").info("getYumConfigFileActiveRfs() called, returning %s", self._yumConfForActiveRfs)
return self._yumConfForActiveRfs
def getYumConfigFileNewVersionOnly (self):
"""
Gets the name of a yum config file, configured for one repository: The one that came with the currently installed version
"""
self._log("get-yum-config-file-new-version-only-called").info("getYumConfigFileNewVersionOnly() called, returning %s", self._yumConfForNewVersionOnly)
return self._yumConfForNewVersionOnly
def getAvailableSize (self, path_):
"""
Returns available size in bytes on a given path
"""
self._log("get-avail-size").info("getAvailableSize() called, path=%s", path_)
return self._utils.getAvailableSize(path_)
def hook_runYumActiveRfs (self, command):
"""Emergency use only. Allows pilot to run any yum command"""
self._log("hook-run-yum-active-rfs-called").info("hook_runYumActiveRfs() called, command=%s", command)
(rc, outText, errText)=self._rpmActiveRfs._runRpm(command)
self._log("hook-run-yum-active-rfs-returning").info("hook_runYumActiveRfs() returning rc=%s, outText=%s, errText=%s",
rc, outText, errText)
return (rc, outText, errText)
def hook_runRpmActiveRfs (self, command):
"""Emergency use only. Allows pilot to run any rpm command"""
self._log("hook-run-rpm-active-rfs-called").info("hook_runRpmActiveRfs() called, command=%s", command)
(rc, outText, errText)=self._yumActiveRfs._runYum(command)
self._log("hook-run-rpm-active-rfs-returning").info("hook_runRpmActiveRfs() returning rc=%s, outText=%s, errText=%s",
rc, outText, errText)
return (rc, outText, errText)
def hook_runCommand (self, command):
"""
Allows pilot to run any command.
Returns (rc, outText, errText)
"""
self._log("hook-run-command-called").info("hook_runCommand() called, command=%s", command)
(rc, outText, errText)=self._utils.runCommand(command)
self._log("hook-run-command-returning").info("hook_runCommand() returning rc=%s", rc)
return (rc, outText, errText)
class SubPkgManager(object):
def __init__(self, logger):
self._log = logger.createLogger("sys-install", "sub-pkg-mng")
self.utils = Utils(self._log)
self.useFakeChrootMode = False
self.rootDir = None
self.subPkgDir = None
self.subPkgTempDir = None
def setDirs(self,
rootDir,
subPkgDir,
subPkgTempDir,
rpmKeysDir,
useFakeChrootMode=False):
self._raiseIfNotAbsPath(subPkgDir, "subPkgDir")
self._raiseIfNotAbsPath(subPkgTempDir, "subPkgTempDir")
self.rootDir = rootDir
self.subPkgDir = subPkgDir
self.subPkgTempDir = subPkgTempDir
self.rpmKeysDir = rpmKeysDir
self.useFakeChrootMode = useFakeChrootMode
self.infoDir = os.path.join(self.subPkgDir, "info")
# During first install, sub-packages dir may not exist
if not os.path.exists(self.subPkgDir):
self.utils.runCommandRaiseIfFail("mkdir -p " + self.subPkgDir)
def findVersionInFile(self, product, r2pmFile):
"""
Expects an r2pm file with exactly one sub-pkg called 'qb-VVV-BBB'.
File must be below root dir
Returns version found in r2pm file (i.e. the string VVV-BBB).
If not exactly one such sub-pkg is found, returns None (="Invalid package file")
"""
self._log("find-version-called").info(
"findVersionInFile() called, product=%s, r2pmFile=%s", product,
r2pmFile)
self._raiseIfNotAbsPath(r2pmFile, "r2pmFile")
# Find version
rpm = Rpm(self._log)
if self.useFakeChrootMode:
rpm.setRoot(self.rootDir, self.useFakeChrootMode)
r2pmFile = self._getFilePathFromRoot(r2pmFile)
files = rpm.doListFiles(r2pmFile)
self._log("find-version-files").info("findVersionInFile(): files=%s",
files)
version_ = None
count_ = 0
for file_ in files:
mo = re.match("^/info/" + product + "-(.*)\.info$", file_)
if mo != None:
version_ = mo.group(1)
count_ += 1
if count_ != 1:
self._log("find-version-bad-count").error(
"findVersionInFile(): File %s has %s sub-pkgs called "
"'%s-...', should have exactly one", r2pmFile, count_, product)
return None
return version_
def add(self, r2pmFile=None, httpRepo=None, packageName=None):
"""
Adds sub-packages, either from a r2pm file or from an http repo.
To install from file:
r2pmFile : file name
To install from an http repo:
httpRepo : repo address
packageName : Name of package containing the sub-packages
Returns a tuple (subPkgs, subPkgstoRemove).
subPkgs: list of sub-pkgs included in this version.
subPkgstoRemove: list of new sub-pkgs (Replaced sub-pkgs are not included in that list). These sub-pkgs
will be removed if 'prepare' operation is canceled
"""
self._log("add-called").info(
"add() called, r2pmFile=%s, httpRepo=%s, packageName=%s", r2pmFile,
httpRepo, packageName)
# Check that our input makes sense
if (r2pmFile == None) == (httpRepo == None):
msg = "add(): bad parameters, must use either r2pmFile or httpRepo"
self._log("add-called-bad").error(msg)
raise InstallException(msg)
if r2pmFile != None:
self._raiseIfNotAbsPath(r2pmFile, "r2pmFile")
else:
if packageName == None:
msg = "add(): bad parameters, httpRepo must come with packageName"
self._log("add-called-bad").error(msg)
raise InstallException(msg)
# Clean the temp dir and create it
if not self.useFakeChrootMode:
# In ut fakechroot mode we do not remove this, because this dir has chroot soft links
self._cleanTempDir()
self.utils.runCommandRaiseIfFail("mkdir -p " + self.subPkgTempDir)
# Import RPM public keys
# In unit tests, it is impossible to get keys from outside of the chroot. Instead, the test does rpm --import
# in this chroot for us.
if not self.useFakeChrootMode:
rpm = Rpm(self._log)
rpm.setRoot(self.subPkgTempDir)
for keyFile in os.listdir(self.rpmKeysDir):
rpm.doImport(os.path.join(self.rpmKeysDir, keyFile))
# Check space - must have at least 4GB free
availableSize = self.utils.getAvailableSize(self.subPkgTempDir)
if availableSize < 4e9:
raise InstallException(
"Internal error: Not enough space on system disk")
# Read all existing sub-pkgs
oldInfoDict = self._readInfoFiles(self.infoDir)
self._log("add-sub-pkg").info("add(): Found old sub-pkgs %s",
oldInfoDict)
oldSubPkgs = oldInfoDict.keys()
# Create temp dir for yum (for config file and such)
if not self.useFakeChrootMode:
yumTempDir = os.path.join(self.subPkgTempDir, "tmp")
else:
yumTempDir = "/tmp"
# Extract r2pm into temp dir
try:
if r2pmFile != None:
# Install from file
removeFile = False
if not self.useFakeChrootMode:
r2pmNameToYum = r2pmFile
# In CentOS6.2, package files must end with '.rpm'
if not r2pmNameToYum.endswith('.rpm'):
r2pmNameToYum = os.path.join(self.subPkgTempDir,
'package.rpm')
self._log("add-sub-pkg-copy").info(
"add(): Copying %s to %s to have a file name that ends with .rpm",
r2pmFile, r2pmNameToYum)
self.utils.runCommandRaiseIfFail(
"cp -f %s %s" % (r2pmFile, r2pmNameToYum))
removeFile = True
else:
self.utils.runCommandRaiseIfFail("cp " + r2pmFile + " " +
self.subPkgTempDir)
r2pmNameToYum = "/" + os.path.basename(r2pmFile)
yum = Yum(self._log, yumTempDir)
yum.setRoot(self.subPkgTempDir,
useFakeChrootMode=self.useFakeChrootMode)
yum.doLocalInstall(r2pmNameToYum)
if removeFile:
self._log("add-sub-pkg-remove").info(
"add(): Removing file that was copied")
self.utils.runCommandRaiseIfFail("rm -f %s" %
r2pmNameToYum)
else:
# Install from http repo
yum = Yum(self._log, yumTempDir)
yum.setRoot(self.subPkgTempDir,
useFakeChrootMode=self.useFakeChrootMode)
yum.addHttpRepo("network", httpRepo)
yum.doInstall(packageName)
# Now, move/replace new sub-pkgs into subPkgDir.
# We do this according to a very specific sequence, so that in case we abort in the middle (power failure or whatever),
# A call to TBD() will restore things the way they were before this operation
# Todo(orens): TBD -> Real function
# Sequence is:
# 1. Copy all existing sub-packages to self.subPkgTempDir, except for those found there already (i.e. new copy
# is better tahn old copy)
# 2. Rename self.subPkgDir with a '.tmp' suffix
# 3. Rename self.subPkgTempDir to become self.subPkgDir
# 4. Remove self.subPkgDir + '.tmp'
# System is always stable, excelt for the split-second between 2 and 3.
# Read list of new sub-packages
newInfoDict = self._readInfoFiles(
os.path.join(self.subPkgTempDir, "info"))
self._log("add-sub-pkg").info("add(): Found new sub-pkgs %s",
newInfoDict)
newSubPkgs = newInfoDict.keys()
# Sanity-check the new info: Make sure new sub-pkgs have all the data we need in their .info files
for subPkg in newSubPkgs:
info = newInfoDict[subPkg]
if not 'dir' in info:
self._log("add-bad-info-dir").error(
"add(): sub-pkg %s info does not have 'dir'", subPkg)
return None
infoDirPath = os.path.join(self.subPkgTempDir, info['dir'])
if not os.path.isdir(infoDirPath):
self._log("add-missing-dir").error(
"add(): sub-pkg %s info has 'dir'->%s, does not exist",
subPkg, infoDirPath)
return None
if not 'name' in info:
self._log("add-bad-info-name").error(
"add(): sub-pkg %s info does not have 'name'", subPkg)
return None
if info['name'] != subPkg:
self._log("add-bad-name").error(
"add(): sub-pkg %s info has 'name'=%s, should be identical",
subPkg, info['name'])
return None
# Step 1: Copy all old sub-packages, which are not found in the new dir, to the new dir
for subPkg in oldSubPkgs:
if subPkg not in newSubPkgs:
oldInfoFile = self._getInfoFileName(self.subPkgDir, subPkg)
newInfoFile = self._getInfoFileName(
self.subPkgTempDir, subPkg)
oldSubPkgDir = os.path.join(self.subPkgDir,
oldInfoDict[subPkg]['dir'])
newSubPkgDir = os.path.join(self.subPkgTempDir,
oldInfoDict[subPkg]['dir'])
if os.path.exists(newSubPkgDir):
self._log("add-bad-name").error(
"add(): old sub-pkg %s not found in new sub-packages, but %s exists",
subPkg, newSubPkgDir)
return None
self.utils.runCommandRaiseIfFail(
"cp -vpf %s %s" % (oldInfoFile, newInfoFile))
self.utils.runCommandRaiseIfFail(
"cp -vrpf %s %s" % (oldSubPkgDir, newSubPkgDir))
# Steps 2-4: Copy all old sub-packages, which are not found in the new dir, to the new dir
tmpOldName = self.subPkgDir + '.tmp'
self.utils.runCommandRaiseIfFail("mv -vf %s %s" %
(self.subPkgDir, tmpOldName))
self.utils.runCommandRaiseIfFail(
"mv -vf %s %s" % (self.subPkgTempDir, self.subPkgDir))
self.utils.runCommandRaiseIfFail("rm -vrf %s" % (tmpOldName))
finally:
# Remove the install-temp directory, no sense in leaving it here
self._cleanTempDir()
# oldInfoDict.keys() holds the list of subPkgs before the install. Let's find out what was added.
newInfoDict = self._readInfoFiles(self.infoDir)
# Sorting makes it easy on unit tests :-)
addedSubPkgs = sorted(
list(set(newInfoDict.keys()) - set(oldInfoDict.keys())))
return (sorted(newSubPkgs), addedSubPkgs)
def removeSubPackages(self, subPackages):
"""
Removes a set of existing sub-pkgs
"""
self._log("remove-sub-packages").info(
"removeSubPackages() called, subPackages=%s", subPackages)
infoDict = self._readInfoFiles(self.infoDir)
for subPkg in subPackages:
if not subPkg in infoDict:
self._log("remove-sub-packages").error(
"removeSubPackages(): sub-package %s not found", subPkg)
continue
subPkgDir = os.path.join(self.subPkgDir, infoDict[subPkg]['dir'])
self.utils.runCommandRaiseIfFail("rm -rf %s" % subPkgDir)
infoFileName = self._getInfoFileName(self.subPkgDir, subPkg)
self.utils.runCommandRaiseIfFail("rm %s" % infoFileName)
def addReposToYum(self, yum, repoFilter=None):
self._log("add-repos-to-yum").info(
"addReposToYum() called, repoFilter=%s", repoFilter)
infoDict = self._readInfoFiles(self.infoDir)
for info in infoDict.keys():
infoData = infoDict[info]
if infoData['type'] == 'repository':
repoDir = os.path.join(self.subPkgDir, infoData['dir'])
repoName = infoData['name']
if (repoFilter is None) or (repoName.find(repoFilter) != -1):
self._log("add-repos-to-yum").info(
"addReposToYum(): Adding repoDir=%s", repoDir)
yum.addDiskRepo(repoName, repoDir)
def getSubPkgs(self):
return self._readInfoFiles(self.infoDir)
def getSubPkgsDir(self):
return self.subPkgDir
def _getFilePathFromRoot(self, r2pmFile):
if not self.useFakeChrootMode:
return r2pmFile
if r2pmFile.startswith(self.rootDir):
return r2pmFile[len(self.rootDir):]
raise ValueError("File %s not in root dir %s" %
(r2pmFile, self.rootDir))
def _readInfoFiles(self, fromDir):
"""
Returns a dict of all info files found in fromDir.
For each file <fromDir>/X.info: X is key, value is a dict with the content of the info file
"""
self._raiseIfNotAbsPath(fromDir, "fromDir")
# No directory ? No sub-pkgs !
if not os.path.exists(fromDir):
return {}
files = os.listdir(fromDir)
ret = {}
for file_ in files:
mo = re.match("^(.*)\.info$", file_)
if mo != None:
subPkg = mo.group(1)
ret[subPkg] = self._readInfoFile(os.path.join(fromDir, file_))
return ret
def _readInfoFile(self, infoFileName):
f = open(infoFileName, "r")
text = f.read()
f.close()
self._log("read-info-file").info(
"_readInfoFile(): Info file %s has text %s", infoFileName, text)
return json.loads(text)
def _getInfoFileName(self, root, subPkg):
return os.path.join(root, "info", subPkg + ".info")
def _cleanTempDir(self):
self.utils.runCommandRaiseIfFail("rm -rf " + self.subPkgTempDir)
def _raiseIfNotAbsPath(self, pathToCheck, name):
if not pathToCheck.startswith('/'):
raise ValueError("%s must start with '/', value given is '%s'" %
(name, pathToCheck))