def cephfs_setup(ctx, config):
testdir = teuthology.get_testdir(ctx)
coverage_dir = '{tdir}/archive/coverage'.format(tdir=testdir)
first_mon = teuthology.get_first_mon(ctx, config)
(mon_remote,) = ctx.cluster.only(first_mon).remotes.iterkeys()
mdss = ctx.cluster.only(teuthology.is_type('mds'))
# If there are any MDSs, then create a filesystem for them to use
# Do this last because requires mon cluster to be up and running
if mdss.remotes:
log.info('Setting up CephFS filesystem...')
ceph_fs = Filesystem(ctx)
if not ceph_fs.legacy_configured():
ceph_fs.create()
is_active_mds = lambda role: role.startswith('mds.') and not role.endswith('-s') and role.find('-s-') == -1
all_roles = [item for remote_roles in mdss.remotes.values() for item in remote_roles]
num_active = len([r for r in all_roles if is_active_mds(r)])
mon_remote.run(args=[
'adjust-ulimits',
'ceph-coverage',
coverage_dir,
'ceph',
'mds', 'set_max_mds', str(num_active)])
yield
def cephfs_setup(ctx, config):
testdir = teuthology.get_testdir(ctx)
coverage_dir = '{tdir}/archive/coverage'.format(tdir=testdir)
first_mon = teuthology.get_first_mon(ctx, config)
(mon_remote, ) = ctx.cluster.only(first_mon).remotes.iterkeys()
mdss = ctx.cluster.only(teuthology.is_type('mds'))
# If there are any MDSs, then create a filesystem for them to use
# Do this last because requires mon cluster to be up and running
if mdss.remotes:
log.info('Setting up CephFS filesystem...')
ceph_fs = Filesystem(ctx)
if not ceph_fs.legacy_configured():
ceph_fs.create()
is_active_mds = lambda role: role.startswith(
'mds.') and not role.endswith('-s') and role.find('-s-') == -1
all_roles = [
item for remote_roles in mdss.remotes.values()
for item in remote_roles
]
num_active = len([r for r in all_roles if is_active_mds(r)])
mon_remote.run(args=[
'sudo', 'adjust-ulimits', 'ceph-coverage', coverage_dir, 'ceph',
'mds', 'set_max_mds',
str(num_active)
])
yield
def cephfs_setup(ctx, config):
cluster_name = config['cluster']
testdir = teuthology.get_testdir(ctx)
coverage_dir = '{tdir}/archive/coverage'.format(tdir=testdir)
first_mon = teuthology.get_first_mon(ctx, config, cluster_name)
(mon_remote,) = ctx.cluster.only(first_mon).remotes.iterkeys()
mdss = ctx.cluster.only(teuthology.is_type('mds', cluster_name))
# If there are any MDSs, then create a filesystem for them to use
# Do this last because requires mon cluster to be up and running
if mdss.remotes:
log.info('Setting up CephFS filesystem...')
ceph_fs = Filesystem(ctx) # TODO: make Filesystem cluster-aware
if not ceph_fs.legacy_configured():
ceph_fs.create()
is_active_mds = lambda role: 'mds.' in role and not role.endswith('-s') and '-s-' not in role
all_roles = [item for remote_roles in mdss.remotes.values() for item in remote_roles]
num_active = len([r for r in all_roles if is_active_mds(r)])
mon_remote.run(
args=[
'sudo',
'adjust-ulimits',
'ceph-coverage',
coverage_dir,
'ceph', 'mds', 'set', 'allow_multimds', 'true',
'--yes-i-really-mean-it'],
check_status=False, # probably old version, upgrade test
)
mon_remote.run(args=[
'sudo',
'adjust-ulimits',
'ceph-coverage',
coverage_dir,
'ceph',
'--cluster', cluster_name,
'mds', 'set_max_mds', str(num_active)])
yield
def cephfs_setup(ctx, config):
cluster_name = config['cluster']
testdir = teuthology.get_testdir(ctx)
coverage_dir = '{tdir}/archive/coverage'.format(tdir=testdir)
first_mon = teuthology.get_first_mon(ctx, config, cluster_name)
(mon_remote,) = ctx.cluster.only(first_mon).remotes.iterkeys()
mdss = ctx.cluster.only(teuthology.is_type('mds', cluster_name))
# If there are any MDSs, then create a filesystem for them to use
# Do this last because requires mon cluster to be up and running
if mdss.remotes:
log.info('Setting up CephFS filesystem...')
ceph_fs = Filesystem(ctx) # TODO: make Filesystem cluster-aware
if not ceph_fs.legacy_configured():
ceph_fs.create()
is_active_mds = lambda role: 'mds.' in role and not role.endswith('-s') and '-s-' not in role
all_roles = [item for remote_roles in mdss.remotes.values() for item in remote_roles]
num_active = len([r for r in all_roles if is_active_mds(r)])
mon_remote.run(
args=[
'sudo',
'adjust-ulimits',
'ceph-coverage',
coverage_dir,
'ceph', 'mds', 'set', 'allow_multimds', 'true',
'--yes-i-really-mean-it'],
check_status=False, # probably old version, upgrade test
)
mon_remote.run(args=[
'sudo',
'adjust-ulimits',
'ceph-coverage',
coverage_dir,
'ceph',
'--cluster', cluster_name,
'mds', 'set_max_mds', str(num_active)])
yield
def build_ceph_cluster(ctx, config):
"""Build a ceph cluster"""
# Expect to find ceph_admin on the first mon by ID, same place that the download task
# puts it. Remember this here, because subsequently IDs will change from those in
# the test config to those that ceph-deploy invents.
(ceph_admin,) = ctx.cluster.only(
teuthology.get_first_mon(ctx, config)).remotes.iterkeys()
def execute_ceph_deploy(cmd):
"""Remotely execute a ceph_deploy command"""
return ceph_admin.run(
args=[
'cd',
'{tdir}/ceph-deploy'.format(tdir=testdir),
run.Raw('&&'),
run.Raw(cmd),
],
check_status=False,
).exitstatus
try:
log.info('Building ceph cluster using ceph-deploy...')
testdir = teuthology.get_testdir(ctx)
ceph_branch = None
if config.get('branch') is not None:
cbranch = config.get('branch')
for var, val in cbranch.iteritems():
ceph_branch = '--{var}={val}'.format(var=var, val=val)
all_nodes = get_all_nodes(ctx, config)
mds_nodes = get_nodes_using_role(ctx, 'mds')
mds_nodes = " ".join(mds_nodes)
mon_node = get_nodes_using_role(ctx, 'mon')
mon_nodes = " ".join(mon_node)
new_mon = './ceph-deploy new' + " " + mon_nodes
mon_hostname = mon_nodes.split(' ')[0]
mon_hostname = str(mon_hostname)
gather_keys = './ceph-deploy gatherkeys' + " " + mon_hostname
deploy_mds = './ceph-deploy mds create' + " " + mds_nodes
no_of_osds = 0
if mon_nodes is None:
raise RuntimeError("no monitor nodes in the config file")
estatus_new = execute_ceph_deploy(new_mon)
if estatus_new != 0:
raise RuntimeError("ceph-deploy: new command failed")
log.info('adding config inputs...')
testdir = teuthology.get_testdir(ctx)
conf_path = '{tdir}/ceph-deploy/ceph.conf'.format(tdir=testdir)
if config.get('conf') is not None:
confp = config.get('conf')
for section, keys in confp.iteritems():
lines = '[{section}]\n'.format(section=section)
teuthology.append_lines_to_file(ceph_admin, conf_path, lines,
sudo=True)
for key, value in keys.iteritems():
log.info("[%s] %s = %s" % (section, key, value))
lines = '{key} = {value}\n'.format(key=key, value=value)
teuthology.append_lines_to_file(
ceph_admin, conf_path, lines, sudo=True)
# install ceph
dev_branch = ctx.config['branch']
branch = '--dev={branch}'.format(branch=dev_branch)
if ceph_branch:
option = ceph_branch
else:
option = branch
install_nodes = './ceph-deploy install ' + option + " " + all_nodes
estatus_install = execute_ceph_deploy(install_nodes)
if estatus_install != 0:
raise RuntimeError("ceph-deploy: Failed to install ceph")
# install ceph-test package too
install_nodes2 = './ceph-deploy install --tests ' + option + \
" " + all_nodes
estatus_install = execute_ceph_deploy(install_nodes2)
if estatus_install != 0:
raise RuntimeError("ceph-deploy: Failed to install ceph-test")
mon_create_nodes = './ceph-deploy mon create-initial'
# If the following fails, it is OK, it might just be that the monitors
# are taking way more than a minute/monitor to form quorum, so lets
# try the next block which will wait up to 15 minutes to gatherkeys.
execute_ceph_deploy(mon_create_nodes)
# create-keys is explicit now
# http://tracker.ceph.com/issues/16036
mons = ctx.cluster.only(teuthology.is_type('mon'))
for remote in mons.remotes.iterkeys():
remote.run(args=['sudo', 'ceph-create-keys', '--cluster', 'ceph',
'--id', remote.shortname])
estatus_gather = execute_ceph_deploy(gather_keys)
if mds_nodes:
estatus_mds = execute_ceph_deploy(deploy_mds)
if estatus_mds != 0:
raise RuntimeError("ceph-deploy: Failed to deploy mds")
if config.get('test_mon_destroy') is not None:
for d in range(1, len(mon_node)):
mon_destroy_nodes = './ceph-deploy mon destroy' + \
" " + mon_node[d]
estatus_mon_d = execute_ceph_deploy(mon_destroy_nodes)
if estatus_mon_d != 0:
raise RuntimeError("ceph-deploy: Failed to delete monitor")
node_dev_list = get_dev_for_osd(ctx, config)
for d in node_dev_list:
node = d[0]
for disk in d[1:]:
zap = './ceph-deploy disk zap ' + node + ':' + disk
estatus = execute_ceph_deploy(zap)
if estatus != 0:
raise RuntimeError("ceph-deploy: Failed to zap osds")
osd_create_cmd = './ceph-deploy osd create '
if config.get('dmcrypt') is not None:
osd_create_cmd += '--dmcrypt '
osd_create_cmd += ":".join(d)
estatus_osd = execute_ceph_deploy(osd_create_cmd)
if estatus_osd == 0:
log.info('successfully created osd')
no_of_osds += 1
else:
raise RuntimeError("ceph-deploy: Failed to create osds")
if config.get('wait-for-healthy', True) and no_of_osds >= 2:
is_healthy(ctx=ctx, config=None)
log.info('Setting up client nodes...')
conf_path = '/etc/ceph/ceph.conf'
admin_keyring_path = '/etc/ceph/ceph.client.admin.keyring'
first_mon = teuthology.get_first_mon(ctx, config)
(mon0_remote,) = ctx.cluster.only(first_mon).remotes.keys()
conf_data = teuthology.get_file(
remote=mon0_remote,
path=conf_path,
sudo=True,
)
admin_keyring = teuthology.get_file(
remote=mon0_remote,
path=admin_keyring_path,
sudo=True,
)
clients = ctx.cluster.only(teuthology.is_type('client'))
for remot, roles_for_host in clients.remotes.iteritems():
for id_ in teuthology.roles_of_type(roles_for_host, 'client'):
client_keyring = \
'/etc/ceph/ceph.client.{id}.keyring'.format(id=id_)
mon0_remote.run(
args=[
'cd',
'{tdir}'.format(tdir=testdir),
run.Raw('&&'),
'sudo', 'bash', '-c',
run.Raw('"'), 'ceph',
'auth',
'get-or-create',
'client.{id}'.format(id=id_),
'mds', 'allow',
'mon', 'allow *',
'osd', 'allow *',
run.Raw('>'),
client_keyring,
run.Raw('"'),
],
)
key_data = teuthology.get_file(
remote=mon0_remote,
path=client_keyring,
sudo=True,
)
teuthology.sudo_write_file(
remote=remot,
path=client_keyring,
data=key_data,
perms='0644'
)
teuthology.sudo_write_file(
remote=remot,
path=admin_keyring_path,
data=admin_keyring,
perms='0644'
)
teuthology.sudo_write_file(
remote=remot,
path=conf_path,
data=conf_data,
perms='0644'
)
if mds_nodes:
log.info('Configuring CephFS...')
ceph_fs = Filesystem(ctx)
if not ceph_fs.legacy_configured():
ceph_fs.create()
elif not config.get('only_mon'):
raise RuntimeError(
"The cluster is NOT operational due to insufficient OSDs")
yield
except Exception:
log.info(
"Error encountered, logging exception before tearing down ceph-deploy")
log.info(traceback.format_exc())
raise
finally:
if config.get('keep_running'):
return
log.info('Stopping ceph...')
ctx.cluster.run(args=['sudo', 'stop', 'ceph-all', run.Raw('||'),
'sudo', 'service', 'ceph', 'stop', run.Raw('||'),
'sudo', 'systemctl', 'stop', 'ceph.target'])
# Are you really not running anymore?
# try first with the init tooling
# ignoring the status so this becomes informational only
ctx.cluster.run(
args=[
'sudo', 'status', 'ceph-all', run.Raw('||'),
'sudo', 'service', 'ceph', 'status', run.Raw('||'),
'sudo', 'systemctl', 'status', 'ceph.target'],
check_status=False)
# and now just check for the processes themselves, as if upstart/sysvinit
# is lying to us. Ignore errors if the grep fails
ctx.cluster.run(args=['sudo', 'ps', 'aux', run.Raw('|'),
'grep', '-v', 'grep', run.Raw('|'),
'grep', 'ceph'], check_status=False)
if ctx.archive is not None:
# archive mon data, too
log.info('Archiving mon data...')
path = os.path.join(ctx.archive, 'data')
os.makedirs(path)
mons = ctx.cluster.only(teuthology.is_type('mon'))
for remote, roles in mons.remotes.iteritems():
for role in roles:
if role.startswith('mon.'):
teuthology.pull_directory_tarball(
remote,
'/var/lib/ceph/mon',
path + '/' + role + '.tgz')
log.info('Compressing logs...')
run.wait(
ctx.cluster.run(
args=[
'sudo',
'find',
'/var/log/ceph',
'-name',
'*.log',
'-print0',
run.Raw('|'),
'sudo',
'xargs',
'-0',
'--no-run-if-empty',
'--',
'gzip',
'--',
],
wait=False,
),
)
log.info('Archiving logs...')
path = os.path.join(ctx.archive, 'remote')
os.makedirs(path)
for remote in ctx.cluster.remotes.iterkeys():
sub = os.path.join(path, remote.shortname)
os.makedirs(sub)
teuthology.pull_directory(remote, '/var/log/ceph',
os.path.join(sub, 'log'))
# Prevent these from being undefined if the try block fails
all_nodes = get_all_nodes(ctx, config)
purge_nodes = './ceph-deploy purge' + " " + all_nodes
purgedata_nodes = './ceph-deploy purgedata' + " " + all_nodes
log.info('Purging package...')
execute_ceph_deploy(purge_nodes)
log.info('Purging data...')
execute_ceph_deploy(purgedata_nodes)
def build_ceph_cluster(ctx, config):
"""Build a ceph cluster"""
# Expect to find ceph_admin on the first mon by ID, same place that the download task
# puts it. Remember this here, because subsequently IDs will change from those in
# the test config to those that ceph-deploy invents.
(ceph_admin,) = ctx.cluster.only(teuthology.get_first_mon(ctx, config)).remotes.iterkeys()
def execute_ceph_deploy(cmd):
"""Remotely execute a ceph_deploy command"""
return ceph_admin.run(
args=[
'cd',
'{tdir}/ceph-deploy'.format(tdir=testdir),
run.Raw('&&'),
run.Raw(cmd),
],
check_status=False,
).exitstatus
try:
log.info('Building ceph cluster using ceph-deploy...')
testdir = teuthology.get_testdir(ctx)
ceph_branch = None
if config.get('branch') is not None:
cbranch = config.get('branch')
for var, val in cbranch.iteritems():
ceph_branch = '--{var}={val}'.format(var=var, val=val)
all_nodes = get_all_nodes(ctx, config)
mds_nodes = get_nodes_using_role(ctx, 'mds')
mds_nodes = " ".join(mds_nodes)
mon_node = get_nodes_using_role(ctx, 'mon')
mon_nodes = " ".join(mon_node)
new_mon = './ceph-deploy new'+" "+mon_nodes
install_nodes = './ceph-deploy install ' + (ceph_branch if ceph_branch else "--dev=master") + " " + all_nodes
mon_hostname = mon_nodes.split(' ')[0]
mon_hostname = str(mon_hostname)
gather_keys = './ceph-deploy gatherkeys'+" "+mon_hostname
deploy_mds = './ceph-deploy mds create'+" "+mds_nodes
no_of_osds = 0
if mon_nodes is None:
raise RuntimeError("no monitor nodes in the config file")
estatus_new = execute_ceph_deploy(new_mon)
if estatus_new != 0:
raise RuntimeError("ceph-deploy: new command failed")
log.info('adding config inputs...')
testdir = teuthology.get_testdir(ctx)
conf_path = '{tdir}/ceph-deploy/ceph.conf'.format(tdir=testdir)
if config.get('conf') is not None:
confp = config.get('conf')
for section, keys in confp.iteritems():
lines = '[{section}]\n'.format(section=section)
teuthology.append_lines_to_file(ceph_admin, conf_path, lines,
sudo=True)
for key, value in keys.iteritems():
log.info("[%s] %s = %s" % (section, key, value))
lines = '{key} = {value}\n'.format(key=key, value=value)
teuthology.append_lines_to_file(ceph_admin, conf_path, lines,
sudo=True)
estatus_install = execute_ceph_deploy(install_nodes)
if estatus_install != 0:
raise RuntimeError("ceph-deploy: Failed to install ceph")
mon_create_nodes = './ceph-deploy mon create-initial'
# If the following fails, it is OK, it might just be that the monitors
# are taking way more than a minute/monitor to form quorum, so lets
# try the next block which will wait up to 15 minutes to gatherkeys.
execute_ceph_deploy(mon_create_nodes)
estatus_gather = execute_ceph_deploy(gather_keys)
max_gather_tries = 90
gather_tries = 0
while (estatus_gather != 0):
gather_tries += 1
if gather_tries >= max_gather_tries:
msg = 'ceph-deploy was not able to gatherkeys after 15 minutes'
raise RuntimeError(msg)
estatus_gather = execute_ceph_deploy(gather_keys)
time.sleep(10)
if mds_nodes:
estatus_mds = execute_ceph_deploy(deploy_mds)
if estatus_mds != 0:
raise RuntimeError("ceph-deploy: Failed to deploy mds")
if config.get('test_mon_destroy') is not None:
for d in range(1, len(mon_node)):
mon_destroy_nodes = './ceph-deploy mon destroy'+" "+mon_node[d]
estatus_mon_d = execute_ceph_deploy(mon_destroy_nodes)
if estatus_mon_d != 0:
raise RuntimeError("ceph-deploy: Failed to delete monitor")
node_dev_list = get_dev_for_osd(ctx, config)
osd_create_cmd = './ceph-deploy osd create --zap-disk '
for d in node_dev_list:
if config.get('dmcrypt') is not None:
osd_create_cmd_d = osd_create_cmd+'--dmcrypt'+" "+d
else:
osd_create_cmd_d = osd_create_cmd+d
estatus_osd = execute_ceph_deploy(osd_create_cmd_d)
if estatus_osd == 0:
log.info('successfully created osd')
no_of_osds += 1
else:
disks = d.split(':')
dev_disk = disks[0]+":"+disks[1]
j_disk = disks[0]+":"+disks[2]
zap_disk = './ceph-deploy disk zap '+dev_disk+" "+j_disk
execute_ceph_deploy(zap_disk)
estatus_osd = execute_ceph_deploy(osd_create_cmd_d)
if estatus_osd == 0:
log.info('successfully created osd')
no_of_osds += 1
else:
raise RuntimeError("ceph-deploy: Failed to create osds")
if config.get('wait-for-healthy', True) and no_of_osds >= 2:
is_healthy(ctx=ctx, config=None)
log.info('Setting up client nodes...')
conf_path = '/etc/ceph/ceph.conf'
admin_keyring_path = '/etc/ceph/ceph.client.admin.keyring'
first_mon = teuthology.get_first_mon(ctx, config)
(mon0_remote,) = ctx.cluster.only(first_mon).remotes.keys()
conf_data = teuthology.get_file(
remote=mon0_remote,
path=conf_path,
sudo=True,
)
admin_keyring = teuthology.get_file(
remote=mon0_remote,
path=admin_keyring_path,
sudo=True,
)
clients = ctx.cluster.only(teuthology.is_type('client'))
for remot, roles_for_host in clients.remotes.iteritems():
for id_ in teuthology.roles_of_type(roles_for_host, 'client'):
client_keyring = \
'/etc/ceph/ceph.client.{id}.keyring'.format(id=id_)
mon0_remote.run(
args=[
'cd',
'{tdir}'.format(tdir=testdir),
run.Raw('&&'),
'sudo', 'bash', '-c',
run.Raw('"'), 'ceph',
'auth',
'get-or-create',
'client.{id}'.format(id=id_),
'mds', 'allow',
'mon', 'allow *',
'osd', 'allow *',
run.Raw('>'),
client_keyring,
run.Raw('"'),
],
)
key_data = teuthology.get_file(
remote=mon0_remote,
path=client_keyring,
sudo=True,
)
teuthology.sudo_write_file(
remote=remot,
path=client_keyring,
data=key_data,
perms='0644'
)
teuthology.sudo_write_file(
remote=remot,
path=admin_keyring_path,
data=admin_keyring,
perms='0644'
)
teuthology.sudo_write_file(
remote=remot,
path=conf_path,
data=conf_data,
perms='0644'
)
log.info('Configuring CephFS...')
ceph_fs = Filesystem(ctx, admin_remote=clients.remotes.keys()[0])
if not ceph_fs.legacy_configured():
ceph_fs.create()
else:
raise RuntimeError(
"The cluster is NOT operational due to insufficient OSDs")
yield
except Exception:
log.info("Error encountered, logging exception before tearing down ceph-deploy")
log.info(traceback.format_exc())
raise
finally:
log.info('Stopping ceph...')
ctx.cluster.run(args=['sudo', 'stop', 'ceph-all', run.Raw('||'),
'sudo', 'service', 'ceph', 'stop' ])
# Are you really not running anymore?
# try first with the init tooling
# ignoring the status so this becomes informational only
ctx.cluster.run(args=['sudo', 'status', 'ceph-all', run.Raw('||'),
'sudo', 'service', 'ceph', 'status'],
check_status=False)
# and now just check for the processes themselves, as if upstart/sysvinit
# is lying to us. Ignore errors if the grep fails
ctx.cluster.run(args=['sudo', 'ps', 'aux', run.Raw('|'),
'grep', '-v', 'grep', run.Raw('|'),
'grep', 'ceph'], check_status=False)
if ctx.archive is not None:
# archive mon data, too
log.info('Archiving mon data...')
path = os.path.join(ctx.archive, 'data')
os.makedirs(path)
mons = ctx.cluster.only(teuthology.is_type('mon'))
for remote, roles in mons.remotes.iteritems():
for role in roles:
if role.startswith('mon.'):
teuthology.pull_directory_tarball(
remote,
'/var/lib/ceph/mon',
path + '/' + role + '.tgz')
log.info('Compressing logs...')
run.wait(
ctx.cluster.run(
args=[
'sudo',
'find',
'/var/log/ceph',
'-name',
'*.log',
'-print0',
run.Raw('|'),
'sudo',
'xargs',
'-0',
'--no-run-if-empty',
'--',
'gzip',
'--',
],
wait=False,
),
)
log.info('Archiving logs...')
path = os.path.join(ctx.archive, 'remote')
os.makedirs(path)
for remote in ctx.cluster.remotes.iterkeys():
sub = os.path.join(path, remote.shortname)
os.makedirs(sub)
teuthology.pull_directory(remote, '/var/log/ceph',
os.path.join(sub, 'log'))
# Prevent these from being undefined if the try block fails
all_nodes = get_all_nodes(ctx, config)
purge_nodes = './ceph-deploy purge'+" "+all_nodes
purgedata_nodes = './ceph-deploy purgedata'+" "+all_nodes
log.info('Purging package...')
execute_ceph_deploy(purge_nodes)
log.info('Purging data...')
execute_ceph_deploy(purgedata_nodes)
def cephfs_setup(ctx, config):
cluster_name = config["cluster"]
testdir = teuthology.get_testdir(ctx)
coverage_dir = "{tdir}/archive/coverage".format(tdir=testdir)
first_mon = teuthology.get_first_mon(ctx, config, cluster_name)
(mon_remote,) = ctx.cluster.only(first_mon).remotes.iterkeys()
mdss = ctx.cluster.only(teuthology.is_type("mds", cluster_name))
# If there are any MDSs, then create a filesystem for them to use
# Do this last because requires mon cluster to be up and running
if mdss.remotes:
log.info("Setting up CephFS filesystem...")
ceph_fs = Filesystem(ctx) # TODO: make Filesystem cluster-aware
if not ceph_fs.legacy_configured():
ceph_fs.create()
is_active_mds = lambda role: "mds." in role and not role.endswith("-s") and "-s-" not in role
all_roles = [item for remote_roles in mdss.remotes.values() for item in remote_roles]
num_active = len([r for r in all_roles if is_active_mds(r)])
mon_remote.run(
args=[
"sudo",
"adjust-ulimits",
"ceph-coverage",
coverage_dir,
"ceph",
"mds",
"set",
"allow_multimds",
"true",
"--yes-i-really-mean-it",
],
check_status=False, # probably old version, upgrade test
)
mon_remote.run(
args=[
"sudo",
"adjust-ulimits",
"ceph-coverage",
coverage_dir,
"ceph",
"--cluster",
cluster_name,
"mds",
"set_max_mds",
str(num_active),
]
)
mon_remote.run(
args=[
"sudo",
"adjust-ulimits",
"ceph-coverage",
coverage_dir,
"ceph",
"mds",
"set",
"allow_dirfrags",
"true",
"--yes-i-really-mean-it",
],
check_status=False, # probably old version, upgrade test
)
yield
def build_ceph_cluster(ctx, config):
"""Build a ceph cluster"""
# Expect to find ceph_admin on the first mon by ID, same place that the download task
# puts it. Remember this here, because subsequently IDs will change from those in
# the test config to those that ceph-deploy invents.
(ceph_admin,) = ctx.cluster.only(teuthology.get_first_mon(ctx, config)).remotes.iterkeys()
def execute_ceph_deploy(cmd):
"""Remotely execute a ceph_deploy command"""
return ceph_admin.run(
args=["cd", "{tdir}/ceph-deploy".format(tdir=testdir), run.Raw("&&"), run.Raw(cmd)], check_status=False
).exitstatus
try:
log.info("Building ceph cluster using ceph-deploy...")
testdir = teuthology.get_testdir(ctx)
ceph_branch = None
if config.get("branch") is not None:
cbranch = config.get("branch")
for var, val in cbranch.iteritems():
ceph_branch = "--{var}={val}".format(var=var, val=val)
all_nodes = get_all_nodes(ctx, config)
mds_nodes = get_nodes_using_role(ctx, "mds")
mds_nodes = " ".join(mds_nodes)
mon_node = get_nodes_using_role(ctx, "mon")
mon_nodes = " ".join(mon_node)
new_mon = "./ceph-deploy new" + " " + mon_nodes
mon_hostname = mon_nodes.split(" ")[0]
mon_hostname = str(mon_hostname)
gather_keys = "./ceph-deploy gatherkeys" + " " + mon_hostname
deploy_mds = "./ceph-deploy mds create" + " " + mds_nodes
no_of_osds = 0
if mon_nodes is None:
raise RuntimeError("no monitor nodes in the config file")
estatus_new = execute_ceph_deploy(new_mon)
if estatus_new != 0:
raise RuntimeError("ceph-deploy: new command failed")
log.info("adding config inputs...")
testdir = teuthology.get_testdir(ctx)
conf_path = "{tdir}/ceph-deploy/ceph.conf".format(tdir=testdir)
if config.get("conf") is not None:
confp = config.get("conf")
for section, keys in confp.iteritems():
lines = "[{section}]\n".format(section=section)
teuthology.append_lines_to_file(ceph_admin, conf_path, lines, sudo=True)
for key, value in keys.iteritems():
log.info("[%s] %s = %s" % (section, key, value))
lines = "{key} = {value}\n".format(key=key, value=value)
teuthology.append_lines_to_file(ceph_admin, conf_path, lines, sudo=True)
# install ceph
install_nodes = "./ceph-deploy install " + (ceph_branch if ceph_branch else "--dev=master") + " " + all_nodes
estatus_install = execute_ceph_deploy(install_nodes)
if estatus_install != 0:
raise RuntimeError("ceph-deploy: Failed to install ceph")
# install ceph-test package too
install_nodes2 = (
"./ceph-deploy install --tests " + (ceph_branch if ceph_branch else "--dev=master") + " " + all_nodes
)
estatus_install = execute_ceph_deploy(install_nodes2)
if estatus_install != 0:
raise RuntimeError("ceph-deploy: Failed to install ceph-test")
mon_create_nodes = "./ceph-deploy mon create-initial"
# If the following fails, it is OK, it might just be that the monitors
# are taking way more than a minute/monitor to form quorum, so lets
# try the next block which will wait up to 15 minutes to gatherkeys.
execute_ceph_deploy(mon_create_nodes)
estatus_gather = execute_ceph_deploy(gather_keys)
max_gather_tries = 90
gather_tries = 0
while estatus_gather != 0:
gather_tries += 1
if gather_tries >= max_gather_tries:
msg = "ceph-deploy was not able to gatherkeys after 15 minutes"
raise RuntimeError(msg)
estatus_gather = execute_ceph_deploy(gather_keys)
time.sleep(10)
if mds_nodes:
estatus_mds = execute_ceph_deploy(deploy_mds)
if estatus_mds != 0:
raise RuntimeError("ceph-deploy: Failed to deploy mds")
if config.get("test_mon_destroy") is not None:
for d in range(1, len(mon_node)):
mon_destroy_nodes = "./ceph-deploy mon destroy" + " " + mon_node[d]
estatus_mon_d = execute_ceph_deploy(mon_destroy_nodes)
if estatus_mon_d != 0:
raise RuntimeError("ceph-deploy: Failed to delete monitor")
node_dev_list = get_dev_for_osd(ctx, config)
for d in node_dev_list:
node = d[0]
for disk in d[1:]:
zap = "./ceph-deploy disk zap " + node + ":" + disk
estatus = execute_ceph_deploy(zap)
if estatus != 0:
raise RuntimeError("ceph-deploy: Failed to zap osds")
osd_create_cmd = "./ceph-deploy osd create "
if config.get("dmcrypt") is not None:
osd_create_cmd += "--dmcrypt "
osd_create_cmd += ":".join(d)
estatus_osd = execute_ceph_deploy(osd_create_cmd)
if estatus_osd == 0:
log.info("successfully created osd")
no_of_osds += 1
else:
raise RuntimeError("ceph-deploy: Failed to create osds")
if config.get("wait-for-healthy", True) and no_of_osds >= 2:
is_healthy(ctx=ctx, config=None)
log.info("Setting up client nodes...")
conf_path = "/etc/ceph/ceph.conf"
admin_keyring_path = "/etc/ceph/ceph.client.admin.keyring"
first_mon = teuthology.get_first_mon(ctx, config)
(mon0_remote,) = ctx.cluster.only(first_mon).remotes.keys()
conf_data = teuthology.get_file(remote=mon0_remote, path=conf_path, sudo=True)
admin_keyring = teuthology.get_file(remote=mon0_remote, path=admin_keyring_path, sudo=True)
clients = ctx.cluster.only(teuthology.is_type("client"))
for remot, roles_for_host in clients.remotes.iteritems():
for id_ in teuthology.roles_of_type(roles_for_host, "client"):
client_keyring = "/etc/ceph/ceph.client.{id}.keyring".format(id=id_)
mon0_remote.run(
args=[
"cd",
"{tdir}".format(tdir=testdir),
run.Raw("&&"),
"sudo",
"bash",
"-c",
run.Raw('"'),
"ceph",
"auth",
"get-or-create",
"client.{id}".format(id=id_),
"mds",
"allow",
"mon",
"allow *",
"osd",
"allow *",
run.Raw(">"),
client_keyring,
run.Raw('"'),
]
)
key_data = teuthology.get_file(remote=mon0_remote, path=client_keyring, sudo=True)
teuthology.sudo_write_file(remote=remot, path=client_keyring, data=key_data, perms="0644")
teuthology.sudo_write_file(remote=remot, path=admin_keyring_path, data=admin_keyring, perms="0644")
teuthology.sudo_write_file(remote=remot, path=conf_path, data=conf_data, perms="0644")
if mds_nodes:
log.info("Configuring CephFS...")
ceph_fs = Filesystem(ctx, admin_remote=clients.remotes.keys()[0])
if not ceph_fs.legacy_configured():
ceph_fs.create()
elif not config.get("only_mon"):
raise RuntimeError("The cluster is NOT operational due to insufficient OSDs")
yield
except Exception:
log.info("Error encountered, logging exception before tearing down ceph-deploy")
log.info(traceback.format_exc())
raise
finally:
if config.get("keep_running"):
return
log.info("Stopping ceph...")
ctx.cluster.run(
args=[
"sudo",
"stop",
"ceph-all",
run.Raw("||"),
"sudo",
"service",
"ceph",
"stop",
run.Raw("||"),
"sudo",
"systemctl",
"stop",
"ceph.target",
]
)
# Are you really not running anymore?
# try first with the init tooling
# ignoring the status so this becomes informational only
ctx.cluster.run(
args=[
"sudo",
"status",
"ceph-all",
run.Raw("||"),
"sudo",
"service",
"ceph",
"status",
run.Raw("||"),
"sudo",
"systemctl",
"status",
"ceph.target",
],
check_status=False,
)
# and now just check for the processes themselves, as if upstart/sysvinit
# is lying to us. Ignore errors if the grep fails
ctx.cluster.run(
args=["sudo", "ps", "aux", run.Raw("|"), "grep", "-v", "grep", run.Raw("|"), "grep", "ceph"],
check_status=False,
)
if ctx.archive is not None:
# archive mon data, too
log.info("Archiving mon data...")
path = os.path.join(ctx.archive, "data")
os.makedirs(path)
mons = ctx.cluster.only(teuthology.is_type("mon"))
for remote, roles in mons.remotes.iteritems():
for role in roles:
if role.startswith("mon."):
teuthology.pull_directory_tarball(remote, "/var/lib/ceph/mon", path + "/" + role + ".tgz")
log.info("Compressing logs...")
run.wait(
ctx.cluster.run(
args=[
"sudo",
"find",
"/var/log/ceph",
"-name",
"*.log",
"-print0",
run.Raw("|"),
"sudo",
"xargs",
"-0",
"--no-run-if-empty",
"--",
"gzip",
"--",
],
wait=False,
)
)
log.info("Archiving logs...")
path = os.path.join(ctx.archive, "remote")
os.makedirs(path)
for remote in ctx.cluster.remotes.iterkeys():
sub = os.path.join(path, remote.shortname)
os.makedirs(sub)
teuthology.pull_directory(remote, "/var/log/ceph", os.path.join(sub, "log"))
# Prevent these from being undefined if the try block fails
all_nodes = get_all_nodes(ctx, config)
purge_nodes = "./ceph-deploy purge" + " " + all_nodes
purgedata_nodes = "./ceph-deploy purgedata" + " " + all_nodes
log.info("Purging package...")
execute_ceph_deploy(purge_nodes)
log.info("Purging data...")
execute_ceph_deploy(purgedata_nodes)
