def dpkg_configure(ctx):
for remote in ctx.cluster.remotes.iterkeys():
if remote.os.package_type != 'deb':
continue
log.info('Waiting for dpkg --configure -a and apt-get -f install...')
remote.run(
args=[
'sudo',
'dpkg',
'--configure',
'-a',
run.Raw(';'),
'sudo',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'--force-yes',
'-f',
'install',
run.Raw('||'),
':',
],
timeout=180,
check_status=False,
)
def synch_clocks(remotes):
for remote in remotes:
remote.run(
args=[
'sudo',
'service',
'ntp',
'stop',
run.Raw('&&'),
'sudo',
'ntpdate-debian',
run.Raw('&&'),
'sudo',
'hwclock',
'--systohc',
'--utc',
run.Raw('&&'),
'sudo',
'service',
'ntp',
'start',
run.Raw('||'),
'true', # ignore errors; we may be racing with ntpd startup
],
timeout=60,
)
def remove_yum_timedhosts(ctx):
# Workaround for https://bugzilla.redhat.com/show_bug.cgi?id=1233329
log.info("Removing yum timedhosts files...")
for remote in ctx.cluster.remotes.iterkeys():
if remote.os.package_type != 'rpm':
continue
remote.run(
args="sudo find /var/cache/yum -name 'timedhosts' -exec rm {} \;",
check_status=False,
)
def undo_multipath(ctx):
"""
Undo any multipath device mappings created, an
remove the packages/daemon that manages them so they don't
come back unless specifically requested by the test.
"""
for remote in ctx.cluster.remotes.iterkeys():
remote.run(args=[
'sudo',
'multipath',
'-F',
],
check_status=False,
timeout=60)
def undo_multipath(ctx):
"""
Undo any multipath device mappings created, an
remove the packages/daemon that manages them so they don't
come back unless specifically requested by the test.
"""
for remote in ctx.cluster.remotes.iterkeys():
remote.run(
args=[
'sudo', 'multipath', '-F',
],
check_status=False,
timeout=60
)
def kill_hadoop(ctx):
for remote in ctx.cluster.remotes.iterkeys():
pids_out = StringIO()
ps_proc = remote.run(args=[
"ps", "-eo", "pid,cmd",
run.Raw("|"), "grep", "java.*hadoop",
run.Raw("|"), "grep", "-v", "grep"
], stdout=pids_out, check_status=False)
if ps_proc.exitstatus == 0:
for line in pids_out.getvalue().strip().split("\n"):
pid, cmdline = line.split(None, 1)
log.info("Killing PID {0} ({1})".format(pid, cmdline))
remote.run(args=["kill", "-9", pid], check_status=False)
def synch_clocks(remotes):
for remote in remotes:
remote.run(
args=[
'sudo', 'service', 'ntp', 'stop',
run.Raw('&&'),
'sudo', 'ntpdate-debian',
run.Raw('&&'),
'sudo', 'hwclock', '--systohc', '--utc',
run.Raw('&&'),
'sudo', 'service', 'ntp', 'start',
run.Raw('||'),
'true', # ignore errors; we may be racing with ntpd startup
],
timeout=60,
)
def remove_testing_tree(ctx):
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'sudo',
'rm',
'-rf',
get_testdir(ctx),
# just for old time's sake
run.Raw('&&'),
'sudo',
'rm',
'-rf',
'/tmp/cephtest',
run.Raw('&&'),
'sudo',
'rm',
'-rf',
'/home/ubuntu/cephtest',
run.Raw('&&'),
'sudo',
'rm',
'-rf',
'/etc/ceph',
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('Waiting for %s to clear filesystem...', name)
proc.exitstatus.get()
def find_kernel_mounts(ctx):
nodes = {}
log.info('Looking for kernel mounts to handle...')
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'grep',
'-q',
' ceph ',
'/etc/mtab',
run.Raw('||'),
'grep',
'-q',
'^/dev/rbd',
'/etc/mtab',
],
wait=False,
)
nodes[remote] = proc
kernel_mounts = list()
for remote, proc in nodes.iteritems():
try:
proc.exitstatus.get()
log.debug('kernel mount exists on %s', remote.name)
kernel_mounts.append(remote)
except run.CommandFailedError: # no mounts!
log.debug('no kernel mount on %s', remote.name)
return kernel_mounts
def dpkg_configure(ctx):
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
if remote.os.package_type != 'deb':
continue
proc = remote.run(
args=[
'sudo',
'dpkg',
'--configure',
'-a',
run.Raw(';'),
'sudo',
'DEBIAN_FRONTEND=noninteractive',
'apt-get',
'-y',
'--force-yes',
'-f',
'install',
run.Raw('||'),
':',
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info(
'Waiting for %s to dpkg --configure -a and apt-get -f install...',
name)
proc.wait()
def synch_clocks(remotes):
nodes = {}
for remote in remotes:
proc = remote.run(
args=[
'sudo',
'service',
'ntp',
'stop',
run.Raw('&&'),
'sudo',
'ntpdate-debian',
run.Raw('&&'),
'sudo',
'hwclock',
'--systohc',
'--utc',
run.Raw('&&'),
'sudo',
'service',
'ntp',
'start',
run.Raw('||'),
'true', # ignore errors; we may be racing with ntpd startup
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('Waiting for clock to synchronize on %s...', name)
proc.exitstatus.get()
def reset_syslog_dir(ctx):
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'if',
'test',
'-e',
'/etc/rsyslog.d/80-cephtest.conf',
run.Raw(';'),
'then',
'sudo',
'rm',
'-f',
'--',
'/etc/rsyslog.d/80-cephtest.conf',
run.Raw('&&'),
'sudo',
'service',
'rsyslog',
'restart',
run.Raw(';'),
'fi',
run.Raw(';'),
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('Waiting for %s to restart syslog...', name)
proc.exitstatus.get()
def dpkg_configure(ctx):
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'sudo',
'dpkg',
'--configure',
'-a',
run.Raw('&&'),
'sudo',
'apt-get',
'-f',
'install',
run.Raw('||'),
':',
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info(
'Waiting for %s to dpkg --configure -a and apt-get -f install...',
name)
proc.exitstatus.get()
def shutdown_daemons(ctx):
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'if', 'grep', '-q', 'ceph-fuse', '/etc/mtab', run.Raw(';'),
'then',
'grep', 'ceph-fuse', '/etc/mtab', run.Raw('|'),
'grep', '-o', " /.* fuse", run.Raw('|'),
'grep', '-o', "/.* ", run.Raw('|'),
'xargs', '-n', '1', 'sudo', 'fusermount', '-u', run.Raw(';'),
'fi',
run.Raw(';'),
'sudo',
'killall',
'--quiet',
'ceph-mon',
'ceph-osd',
'ceph-mds',
'ceph-fuse',
'ceph-disk',
'radosgw',
'ceph_test_rados',
'rados',
'apache2',
run.Raw('||'),
'true', # ignore errors from ceph binaries not being found
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('Waiting for %s to finish shutdowns...', name)
proc.wait()
def reboot(ctx, remotes):
nodes = {}
for remote in remotes:
log.info('rebooting %s', remote.name)
try:
proc = remote.run( # note use of -n to force a no-sync reboot
args=[
'sync',
run.Raw('&'),
'sleep', '5',
run.Raw(';'),
'sudo', 'reboot', '-f', '-n'
],
wait=False
)
except Exception:
log.exception('ignoring exception during reboot command')
nodes[remote] = proc
# we just ignore these procs because reboot -f doesn't actually
# send anything back to the ssh client!
# for remote, proc in nodes.iteritems():
# proc.wait()
if remotes:
log.info('waiting for nodes to reboot')
time.sleep(8) # if we try and reconnect too quickly, it succeeds!
reconnect(ctx, 480) # allow 8 minutes for the reboots
def shutdown_daemons(ctx):
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'if', 'grep', '-q', 'ceph-fuse', '/etc/mtab', run.Raw(';'),
'then',
'grep', 'ceph-fuse', '/etc/mtab', run.Raw('|'),
'grep', '-o', " /.* fuse", run.Raw('|'),
'grep', '-o', "/.* ", run.Raw('|'),
'xargs', 'sudo', 'fusermount', '-u', run.Raw(';'),
'fi',
run.Raw(';'),
'sudo',
'killall',
'--quiet',
'ceph-mon',
'ceph-osd',
'ceph-mds',
'ceph-fuse',
'ceph-disk',
'radosgw',
'ceph_test_rados',
'rados',
'apache2',
run.Raw('||'),
'true', # ignore errors from ceph binaries not being found
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('Waiting for %s to finish shutdowns...', name)
proc.wait()
def dpkg_configure(ctx):
for remote in ctx.cluster.remotes.iterkeys():
if remote.os.package_type != 'deb':
continue
log.info(
'Waiting for dpkg --configure -a and apt-get -f install...')
remote.run(
args=[
'sudo', 'dpkg', '--configure', '-a',
run.Raw(';'),
'sudo', 'DEBIAN_FRONTEND=noninteractive',
'apt-get', '-y', '--force-yes', '-f', 'install',
run.Raw('||'),
':',
],
timeout=180,
check_status=False,
)
def undo_multipath(ctx):
"""
Undo any multipath device mappings created, an
remove the packages/daemon that manages them so they don't
come back unless specifically requested by the test.
"""
for remote in ctx.cluster.remotes.iterkeys():
remote.run(
args=[
'sudo', 'multipath', '-F',
],
check_status=False,
)
install_task.remove_packages(
ctx,
dict(), # task config not relevant here
{
"rpm": ['multipath-tools', 'device-mapper-multipath'],
"deb": ['multipath-tools'],
}
)
def undo_multipath(ctx):
"""
Undo any multipath device mappings created, an
remove the packages/daemon that manages them so they don't
come back unless specifically requested by the test.
"""
for remote in ctx.cluster.remotes.iterkeys():
remote.run(
args=[
'sudo', 'multipath', '-F',
],
check_status=False,
)
install_task.remove_packages(
ctx,
dict(), # task config not relevant here
{
"rpm": ['multipath-tools', 'device-mapper-multipath'],
"deb": ['multipath-tools'],
}
)
def dpkg_configure(ctx):
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'sudo', 'dpkg', '--configure', '-a',
run.Raw('&&'),
'sudo', 'apt-get', '-f', 'install',
run.Raw('||'),
':',
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info(
'Waiting for %s to dpkg --configure -a and apt-get -f install...',
name)
proc.wait()
def remove_kernel_mounts(ctx, kernel_mounts):
"""
properly we should be able to just do a forced unmount,
but that doesn't seem to be working, so you should reboot instead
"""
nodes = {}
for remote in kernel_mounts:
log.info('clearing kernel mount from %s', remote.name)
proc = remote.run(args=[
'grep', 'ceph', '/etc/mtab',
run.Raw('|'), 'grep', '-o', "on /.* type",
run.Raw('|'), 'grep', '-o', "/.* ",
run.Raw('|'), 'xargs', '-r', 'sudo', 'umount', '-f',
run.Raw(';'), 'fi'
],
wait=False)
nodes[remote] = proc
for remote, proc in nodes:
proc.exitstatus.get()
def synch_clocks(remotes):
nodes = {}
for remote in remotes:
proc = remote.run(
args=[
'sudo', 'service', 'ntp', 'stop',
run.Raw('&&'),
'sudo', 'ntpdate-debian',
run.Raw('&&'),
'sudo', 'hwclock', '--systohc', '--utc',
run.Raw('&&'),
'sudo', 'service', 'ntp', 'start',
run.Raw('||'),
'true', # ignore errors; we may be racing with ntpd startup
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('Waiting for clock to synchronize on %s...', name)
proc.wait()
def remove_testing_tree(ctx):
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'sudo', 'rm', '-rf', get_testdir(ctx),
# just for old time's sake
run.Raw('&&'),
'sudo', 'rm', '-rf', '/tmp/cephtest',
run.Raw('&&'),
'sudo', 'rm', '-rf', '/home/ubuntu/cephtest',
run.Raw('&&'),
'sudo', 'rm', '-rf', '/etc/ceph',
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('Waiting for %s to clear filesystem...', name)
proc.wait()
def reset_syslog_dir(ctx):
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'if', 'test', '-e', '/etc/rsyslog.d/80-cephtest.conf',
run.Raw(';'),
'then',
'sudo', 'rm', '-f', '--', '/etc/rsyslog.d/80-cephtest.conf',
run.Raw('&&'),
'sudo', 'service', 'rsyslog', 'restart',
run.Raw(';'),
'fi',
run.Raw(';'),
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('Waiting for %s to restart syslog...', name)
proc.wait()
def remove_configuration_files(ctx):
"""
Goes through a list of commonly used configuration files used for testing
that should not be left behind.
For example, sometimes ceph-deploy may be configured via
``~/.cephdeploy.conf`` to alter how it handles installation by specifying
a default section in its config with custom locations.
"""
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=['rm', '-f', '/home/ubuntu/.cephdeploy.conf'],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('removing temporary configuration files on %s', name)
proc.wait()
def dpkg_configure(ctx):
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
if remote.os.package_type != 'deb':
continue
proc = remote.run(
args=[
'sudo', 'dpkg', '--configure', '-a',
run.Raw(';'),
'sudo', 'DEBIAN_FRONTEND=noninteractive',
'apt-get', '-y', '--force-yes', '-f', 'install',
run.Raw('||'),
':',
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info(
'Waiting for %s to dpkg --configure -a and apt-get -f install...',
name)
proc.wait()
def remove_kernel_mounts(ctx, kernel_mounts):
"""
properly we should be able to just do a forced unmount,
but that doesn't seem to be working, so you should reboot instead
"""
nodes = {}
for remote in kernel_mounts:
log.info('clearing kernel mount from %s', remote.name)
proc = remote.run(
args=[
'grep', 'ceph', '/etc/mtab', run.Raw('|'),
'grep', '-o', "on /.* type", run.Raw('|'),
'grep', '-o', "/.* ", run.Raw('|'),
'xargs', '-r',
'sudo', 'umount', '-f', run.Raw(';'),
'fi'
],
wait=False
)
nodes[remote] = proc
for remote, proc in nodes:
proc.wait()
def find_kernel_mounts(ctx):
nodes = {}
log.info('Looking for kernel mounts to handle...')
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'grep', '-q', ' ceph ', '/etc/mtab',
run.Raw('||'),
'grep', '-q', '^/dev/rbd', '/etc/mtab',
],
wait=False,
)
nodes[remote] = proc
kernel_mounts = list()
for remote, proc in nodes.iteritems():
try:
proc.wait()
log.debug('kernel mount exists on %s', remote.name)
kernel_mounts.append(remote)
except run.CommandFailedError: # no mounts!
log.debug('no kernel mount on %s', remote.name)
return kernel_mounts
def remove_configuration_files(ctx):
"""
Goes through a list of commonly used configuration files used for testing
that should not be left behind.
For example, sometimes ceph-deploy may be configured via
``~/.cephdeploy.conf`` to alter how it handles installation by specifying
a default section in its config with custom locations.
"""
nodes = {}
for remote in ctx.cluster.remotes.iterkeys():
proc = remote.run(
args=[
'rm', '-f', '/home/ubuntu/.cephdeploy.conf'
],
wait=False,
)
nodes[remote.name] = proc
for name, proc in nodes.iteritems():
log.info('removing temporary configuration files on %s', name)
proc.wait()
def remove_ceph_packages(ctx):
"""
remove ceph and ceph dependent packages by force
force is needed since the node's repo might have changed and
in many cases autocorrect will not work due to missing packages
due to repo changes
"""
ceph_packages_to_remove = [
'ceph-common', 'ceph-mon', 'ceph-osd', 'libcephfs1', 'librados2',
'librgw2', 'librbd1', 'ceph-selinux', 'python-cephfs', 'ceph-base',
'python-rbd', 'python-rados', 'ceph-mds', 'libcephfs-java',
'libcephfs-jni', 'ceph-deploy', 'libapache2-mod-fastcgi'
]
pkgs = str.join(' ', ceph_packages_to_remove)
for remote in ctx.cluster.remotes.iterkeys():
if remote.os.package_type == 'rpm':
log.info("Remove any broken repos")
remote.run(args=['sudo', 'rm',
run.Raw("/etc/yum.repos.d/*ceph*")],
check_status=False)
remote.run(
args=['sudo', 'rm',
run.Raw("/etc/yum.repos.d/*fcgi*")],
check_status=False,
)
remote.run(args=[
'sudo', 'rpm', '--rebuilddb',
run.Raw('&&'), 'yum', 'clean', 'all'
])
log.info('Remove any ceph packages')
remote.run(args=['sudo', 'yum', 'remove', '-y',
run.Raw(pkgs)],
check_status=False)
else:
log.info("Remove any broken repos")
remote.run(
args=['sudo', 'rm',
run.Raw("/etc/apt/sources.list.d/*ceph*")],
check_status=False,
)
log.info("Autoclean")
remote.run(
args=['sudo', 'apt-get', 'autoclean'],
check_status=False,
)
log.info('Remove any ceph packages')
remote.run(args=[
'sudo', 'dpkg', '--remove', '--force-remove-reinstreq',
run.Raw(pkgs)
],
check_status=False)
log.info("Autoclean")
remote.run(args=['sudo', 'apt-get', 'autoclean'])
def remove_ceph_packages(ctx):
"""
remove ceph and ceph dependent packages by force
force is needed since the node's repo might have changed and
in many cases autocorrect will not work due to missing packages
due to repo changes
"""
ceph_packages_to_remove = ['ceph-common', 'ceph-mon', 'ceph-osd',
'libcephfs1', 'librados2', 'librgw2', 'librbd1',
'ceph-selinux', 'python-cephfs', 'ceph-base',
'python-rbd', 'python-rados', 'ceph-mds',
'libcephfs-java', 'libcephfs-jni',
'ceph-deploy', 'libapache2-mod-fastcgi'
]
pkgs = str.join(' ', ceph_packages_to_remove)
for remote in ctx.cluster.remotes.iterkeys():
if remote.os.package_type == 'rpm':
log.info("Remove any broken repos")
remote.run(
args=['sudo', 'rm', run.Raw("/etc/yum.repos.d/*ceph*")],
check_status=False
)
remote.run(
args=['sudo', 'rm', run.Raw("/etc/yum.repos.d/*fcgi*")],
check_status=False,
)
remote.run(
args=['sudo', 'rpm', '--rebuilddb', run.Raw('&&'), 'yum',
'clean', 'all']
)
log.info('Remove any ceph packages')
remote.run(
args=['sudo', 'yum', 'remove', '-y', run.Raw(pkgs)],
check_status=False
)
else:
log.info("Remove any broken repos")
remote.run(
args=['sudo', 'rm', run.Raw("/etc/apt/sources.list.d/*ceph*")],
check_status=False,
)
log.info("Autoclean")
remote.run(
args=['sudo', 'apt-get', 'autoclean'],
check_status=False,
)
log.info('Remove any ceph packages')
remote.run(
args=[
'sudo', 'dpkg', '--remove', '--force-remove-reinstreq',
run.Raw(pkgs)
],
check_status=False
)
log.info("Autoclean")
remote.run(
args=['sudo', 'apt-get', 'autoclean']
)