def verify_package_version(ctx, config, remote):
"""
Ensures that the version of package installed is what
was asked for in the config.
For most cases this is for ceph, but we also install samba
for example.
"""
# Do not verify the version if the ceph-deploy task is being used to
# install ceph. Verifying the ceph installed by ceph-deploy should work,
# but the qa suites will need reorganized first to run ceph-deploy
# before the install task.
# see: http://tracker.ceph.com/issues/11248
if config.get("extras"):
log.info("Skipping version verification...")
return True
if 'repos' in config and config.get('repos'):
log.info(
"Skipping version verification because we have custom repos...")
return True
builder = _get_builder_project(ctx, remote, config)
version = builder.version
pkg_to_check = builder.project
installed_ver = packaging.get_package_version(remote, pkg_to_check)
if installed_ver and version in installed_ver:
msg = "The correct {pkg} version {ver} is installed.".format(
ver=version, pkg=pkg_to_check)
log.info(msg)
else:
raise RuntimeError(
"{pkg} version {ver} was not installed, found {installed}.".format(
ver=version, installed=installed_ver, pkg=pkg_to_check))
def _remove_sources_list(ctx, config, remote):
builder = _get_builder_project(ctx, remote, config)
builder.remove_repo()
remote.run(
args=[
'sudo', 'apt-get', 'update',
],
check_status=False,
)
def _remove_sources_list(ctx, config, remote):
"""
Removes /etc/yum.repos.d/{proj}.repo
:param remote: the teuthology.orchestra.remote.Remote object
:param proj: the project whose .repo needs removing
"""
builder = _get_builder_project(ctx, remote, config)
builder.remove_repo()
if remote.os.name not in ['opensuse', 'sle']:
_yum_unset_check_obsoletes(remote)
def _remove(ctx, config, remote, rpm):
"""
Removes RPM packages from remote
:param ctx: the argparse.Namespace object
:param config: the config dict
:param remote: the teuthology.orchestra.remote.Remote object
:param rpm: list of packages names to remove
"""
remote_os = remote.os
dist_release = remote_os.name
rpm = _package_overrides(rpm, remote_os)
install_ceph_packages = config.get('install_ceph_packages')
if install_ceph_packages:
log.info("Removing packages: {pkglist} on rpm system.".format(
pkglist=", ".join(rpm)))
if dist_release in ['opensuse', 'sle']:
remote.run(args='''
for d in {rpms} ; do
sudo zypper -n --no-gpg-checks remove --capability $d || true
done'''.format(rpms=' '.join(rpm)))
remote.run(args='sudo zypper clean -a')
else:
remote.run(args='''
for d in {rpms} ; do
sudo yum -y remove $d || true
done'''.format(rpms=' '.join(rpm)))
remote.run(args='sudo yum clean all')
else:
log.info("install task did not install any packages, "
"so not removing any, either")
repos = config.get('repos')
if repos:
if dist_release in ['opensuse', 'sle']:
_zypper_removerepo(remote, repos)
else:
raise Exception('Custom repos were specified for %s ' % remote_os +
'but these are currently not supported')
else:
builder = _get_builder_project(ctx, remote, config)
builder.remove_repo()
if dist_release in ['opensuse', 'sle']:
#remote.run(args='sudo zypper clean -a')
log.info(
"Not cleaning zypper cache: this might fail, and is not needed "
"because the test machine will be destroyed or reimaged anyway")
else:
remote.run(args='sudo yum clean expire-cache')
def _upgrade_packages(ctx, config, remote, debs):
"""
Upgrade project's packages on remote Debian host
Before doing so, installs the project's GPG key, writes a sources.list
file, and runs ``apt-get update``.
:param ctx: the argparse.Namespace object
:param config: the config dict
:param remote: the teuthology.orchestra.remote.Remote object
:param debs: the Debian packages to be installed
:param branch: the branch of the project to be used
"""
# check for ceph release key
r = remote.run(
args=[
'sudo', 'apt-key', 'list', run.Raw('|'), 'grep', 'Ceph',
],
stdout=StringIO(),
check_status=False,
)
if r.stdout.getvalue().find('Ceph automated package') == -1:
# if it doesn't exist, add it
remote.run(
args=[
'wget', '-q', '-O-',
'http://git.ceph.com/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc', # noqa
run.Raw('|'),
'sudo', 'apt-key', 'add', '-',
],
stdout=StringIO(),
)
builder = _get_builder_project(ctx, remote, config)
base_url = builder.base_url
log.info('Pulling from %s', base_url)
version = builder.version
log.info('Package version is %s', version)
builder.install_repo()
remote.run(args=['sudo', 'apt-get', 'update'], check_status=False)
install_dep_packages(remote,
args=[
'sudo',
'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y', '--force-yes',
'-o', run.Raw('Dpkg::Options::="--force-confdef"'), '-o', run.Raw(
'Dpkg::Options::="--force-confold"'),
'install',
] + ['%s=%s' % (d, version) for d in debs],
)
def _remove_sources_list(ctx, config, remote):
"""
Removes /etc/yum.repos.d/{proj}.repo, /var/lib/{proj}, and /var/log/{proj}
:param remote: the teuthology.orchestra.remote.Remote object
:param proj: the project whose .repo needs removing
"""
builder = _get_builder_project(ctx, remote, config)
builder.remove_repo()
proj = builder.project
# FIXME
# There probably should be a way of removing these files that is
# implemented in the yum/rpm remove procedures for the ceph package.
# FIXME but why is this function doing these things?
remote.run(
args=['sudo', 'rm', '-r', '/var/lib/{proj}'.format(proj=proj)],
check_status=False,
)
remote.run(
args=['sudo', 'rm', '-r', '/var/log/{proj}'.format(proj=proj)],
check_status=False,
)
if remote.os.name not in ['opensuse', 'sle']:
_yum_unset_check_obsoletes(remote)
def get_upgrade_version(ctx, config, remote):
builder = _get_builder_project(ctx, remote, config)
version = builder.version
return version
def _upgrade_packages(ctx, config, remote, pkgs):
"""
Upgrade project's packages on remote RPM-based host
Before doing so, it makes sure the project's repository is installed -
removing any previous version first.
:param ctx: the argparse.Namespace object
:param config: the config dict
:param remote: the teuthology.orchestra.remote.Remote object
:param pkgs: the RPM packages to be installed
:param branch: the branch of the project to be used
"""
builder = _get_builder_project(ctx, remote, config)
log.info(
"Host {host} is: {distro} {ver} {arch}".format(
host=remote.shortname,
distro=builder.os_type,
ver=builder.os_version,
arch=builder.arch,)
)
base_url = builder.base_url
log.info('Repo base URL: %s', base_url)
project = builder.project
# Remove the repository before re-adding it
builder.remove_repo()
builder.install_repo()
if builder.dist_release not in ['opensuse', 'sle']:
uri = builder.uri_reference
_yum_fix_repo_priority(remote, project, uri)
_yum_fix_repo_host(remote, project)
_yum_set_check_obsoletes(remote)
if builder.dist_release in ['opensuse', 'sle']:
pkg_mng_cmd = 'zypper'
pkg_mng_opts = '-a'
else:
pkg_mng_cmd = 'yum'
pkg_mng_opts = 'all'
remote.run(
args=[
'sudo', pkg_mng_cmd, 'clean', pkg_mng_opts,
])
# Actually upgrade the project packages
if builder.dist_release in ['opensuse', 'sle']:
pkg_mng_opts = '-n'
pkg_mng_subcommand = 'install'
pkg_mng_subcommand_opts = ['--capability', '--no-recommends']
else:
pkg_mng_opts = '-y'
pkg_mng_subcommand = 'upgrade'
pkg_mng_subcommand_opts = []
args = ['sudo', pkg_mng_cmd, pkg_mng_opts, pkg_mng_subcommand]
if pkg_mng_subcommand_opts:
args += pkg_mng_subcommand_opts
args += pkgs
remote.run(args=args)
def _update_package_list_and_install(ctx, remote, rpm, config):
"""
Installs the repository for the relevant branch, then installs
the requested packages on the remote system.
TODO: split this into at least two functions.
:param ctx: the argparse.Namespace object
:param remote: the teuthology.orchestra.remote.Remote object
:param rpm: list of packages names to install
:param config: the config dict
"""
# rpm does not force installation of a particular version of the project
# packages, so we can put extra_system_packages together with the rest
system_pkglist = config.get('extra_system_packages')
if system_pkglist:
if isinstance(system_pkglist, dict):
rpm += system_pkglist.get('rpm')
else:
rpm += system_pkglist
remote_os = remote.os
dist_release = remote_os.name
log.debug("_update_package_list_and_install: config is {}".format(config))
repos = config.get('repos')
install_ceph_packages = config.get('install_ceph_packages')
repos_only = config.get('repos_only')
if repos:
log.debug("Adding repos: %s" % repos)
if dist_release in ['opensuse', 'sle']:
_zypper_wipe_all_repos(remote)
_zypper_addrepo(remote, repos)
else:
raise Exception('Custom repos were specified for %s ' % remote_os +
'but these are currently not supported')
else:
builder = _get_builder_project(ctx, remote, config)
log.info('Pulling from %s', builder.base_url)
log.info('Package version is %s', builder.version)
builder.install_repo()
if repos_only:
log.info("repos_only was specified: not installing any packages")
return None
if not install_ceph_packages:
log.info("install_ceph_packages set to False: not installing Ceph packages")
# Although "librados2" is an indirect dependency of ceph-test, we
# install it separately because, otherwise, ceph-test cannot be
# installed (even with --force) when there are several conflicting
# repos from different vendors.
rpm = ["librados2", "ceph-test"]
# rpm does not force installation of a particular version of the project
# packages, so we can put extra_system_packages together with the rest
system_pkglist = config.get('extra_system_packages', [])
if system_pkglist:
if isinstance(system_pkglist, dict):
rpm += system_pkglist.get('rpm')
else:
rpm += system_pkglist
log.info("Installing packages: {pkglist} on remote rpm {arch}".format(
pkglist=", ".join(rpm), arch=remote.arch))
if dist_release not in ['opensuse', 'sle']:
project = builder.project
uri = builder.uri_reference
_yum_fix_repo_priority(remote, project, uri)
_yum_fix_repo_host(remote, project)
_yum_set_check_obsoletes(remote)
if dist_release in ['opensuse', 'sle']:
remote.run(args='sudo zypper clean -a')
else:
remote.run(args='sudo yum clean all')
ldir = _get_local_dir(config, remote)
if dist_release in ['opensuse', 'sle']:
remove_cmd = 'sudo zypper -n remove --capability'
# NOTE: --capability contradicts --force
install_cmd = 'sudo zypper -n --no-gpg-checks install --force --no-recommends'
else:
remove_cmd = 'sudo yum -y remove'
install_cmd = 'sudo yum -y install'
# to compose version string like "0.94.10-87.g116a558.el7"
pkg_version = '.'.join([builder.version, builder.dist_release])
rpm = _downgrade_packages(ctx, remote, rpm, pkg_version, config)
if system_pkglist:
_retry_if_failures_are_recoverable(remote,
args='{install_cmd} {rpms}'
.format(install_cmd=install_cmd, rpms=' '.join(rpm))
)
else:
for cpack in rpm:
if ldir:
_retry_if_failures_are_recoverable(remote,
args='''
if test -e {pkg} ; then
{remove_cmd} {pkg} ;
{install_cmd} {pkg} ;
else
{install_cmd} {cpack} ;
fi
'''.format(remove_cmd=remove_cmd,
install_cmd=install_cmd,
pkg=os.path.join(ldir, cpack),
cpack=cpack))
else:
_retry_if_failures_are_recoverable(remote,
args='{install_cmd} {cpack}'
.format(install_cmd=install_cmd, cpack=cpack)
)
def _update_package_list_and_install(ctx, remote, debs, config):
"""
Runs ``apt-get update`` first, then runs ``apt-get install``, installing
the requested packages on the remote system.
TODO: split this into at least two functions.
:param ctx: the argparse.Namespace object
:param remote: the teuthology.orchestra.remote.Remote object
:param debs: list of packages names to install
:param config: the config dict
"""
# check for ceph release key
r = remote.run(
args=[
'sudo', 'apt-key', 'list', run.Raw('|'), 'grep', 'Ceph',
],
stdout=StringIO(),
check_status=False,
)
if r.stdout.getvalue().find('Ceph automated package') == -1:
# if it doesn't exist, add it
remote.run(
args=[
'wget', '-q', '-O-',
'http://git.ceph.com/?p=ceph.git;a=blob_plain;f=keys/autobuild.asc', # noqa
run.Raw('|'),
'sudo', 'apt-key', 'add', '-',
],
stdout=StringIO(),
)
builder = _get_builder_project(ctx, remote, config)
log.info("Installing packages: {pkglist} on remote deb {arch}".format(
pkglist=", ".join(debs), arch=builder.arch)
)
system_pkglist = config.get('extra_system_packages')
if system_pkglist:
if isinstance(system_pkglist, dict):
system_pkglist = system_pkglist.get('deb')
log.info("Installing system (non-project) packages: {pkglist} on remote deb {arch}".format(
pkglist=", ".join(system_pkglist), arch=builder.arch)
)
# get baseurl
log.info('Pulling from %s', builder.base_url)
version = builder.version
log.info('Package version is %s', version)
builder.install_repo()
remote.run(args=['sudo', 'apt-get', 'update'], check_status=False)
install_cmd = [
'sudo', 'DEBIAN_FRONTEND=noninteractive', 'apt-get', '-y',
'--force-yes',
'-o', run.Raw('Dpkg::Options::="--force-confdef"'), '-o', run.Raw(
'Dpkg::Options::="--force-confold"'),
'install',
]
install_dep_packages(remote,
args=install_cmd + ['%s=%s' % (d, version) for d in debs],
)
if system_pkglist:
install_dep_packages(remote,
args=install_cmd + system_pkglist,
)
ldir = _get_local_dir(config, remote)
if ldir:
for fyle in os.listdir(ldir):
fname = "%s/%s" % (ldir, fyle)
remote.run(args=['sudo', 'dpkg', '-i', fname],)
