def create_instance(self, flavor_name='test_flavor', ram=64, vcpus=1,
disk=1, server_name='test_instance', image_name=None,
neutron_network=True, label=None):
LOGGER.debug('Try to create instance')
start_time = time.time()
while time.time() - start_time < 100:
try:
if image_name:
image = [i.id for i in self.nova.images.list()
if i.name == image_name]
else:
image = [i.id for i in self.nova.images.list()]
break
except:
pass
else:
raise Exception('Can not get image')
kwargs = {}
if neutron_network:
net_label = label if label else 'net04'
network = self.nova.networks.find(label=net_label)
kwargs['nics'] = [{'net-id': network.id, 'v4-fixed-ip': ''}]
LOGGER.info('image uuid is {0}'.format(image))
flavor = self.nova.flavors.create(
name=flavor_name, ram=ram, vcpus=vcpus, disk=disk)
LOGGER.info('flavor is {0}'.format(flavor.name))
server = self.nova.servers.create(
name=server_name, image=image[0], flavor=flavor, **kwargs)
LOGGER.info('server is {0}'.format(server.name))
return server
def modify_resolv_conf(self, nameservers=None, merge=True):
if nameservers is None:
nameservers = []
resolv_conf = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd='cat /etc/resolv.conf'
)
assert_equal(0, resolv_conf['exit_code'],
'Executing "{0}" on the admin node has failed with: {1}'
.format('cat /etc/resolv.conf', resolv_conf['stderr']))
if merge:
nameservers.extend(resolv_conf['stdout'])
resolv_keys = ['search', 'domain', 'nameserver']
resolv_new = "".join('{0}\n'.format(ns) for ns in nameservers
if any(x in ns for x in resolv_keys))
logger.debug('echo "{0}" > /etc/resolv.conf'.format(resolv_new))
echo_cmd = 'echo "{0}" > /etc/resolv.conf'.format(resolv_new)
echo_result = self.ssh_manager.execute(
ip=self.ssh_manager.admin_ip,
cmd=echo_cmd
)
assert_equal(0, echo_result['exit_code'],
'Executing "{0}" on the admin node has failed with: {1}'
.format(echo_cmd, echo_result['stderr']))
return resolv_conf['stdout']
def _store_file_list(self):
r = self._request_file_list()
text = r.text
files = utils.filter_response_text(text)
logger.debug('Changed files list {}'.format(files))
self._files_list.update(set(filter(lambda x: x != '/COMMIT_MSG',
utils.json_to_dict(files).keys())))
def replace_package(remote, package_name, package_path):
cmd = "ls -all {0} | grep noarch.rpm| awk '{{print $9}}' ".format(
package_path)
result = remote.check_call(cmd, error_info='Failed package replace.')
package_from_review = result.stdout_str
income_version = get_package_version(
remote, os.path.join(package_path, package_from_review),
income=True)
logger.info('Version of package from review {0}'.format(
income_version))
installed_rpm = get_package_version(
remote, package_name)
logger.info('Version of installed package{0}'.format(installed_rpm))
if installed_rpm != income_version:
logger.info('Try to install package {0}'.format(
package_from_review))
cmd = 'rpm -Uvh --oldpackage {0}{1}*.noarch.rpm'.format(
package_path, package_name)
install_result = remote.execute(cmd)
logger.debug('Install package result {0}'.format(install_result))
installed_rpm = get_package_version(
remote, package_name)
asserts.assert_equal(
installed_rpm, income_version,
'Package {0} from review '
'installation fails. Current installed '
'package is {1}'.format(income_version, installed_rpm))
def test_check_nodes_disks(self):
"""Verify hard drive sizes for deployed nodes
Scenario:
1. Create cluster
2. Add 1 controller
3. Add 1 compute
4. Add 1 cinder
5. Deploy cluster
6. Verify hard drive sizes for deployed nodes
7. Run network verify
8. Run OSTF
Duration 15m
"""
cluster_id = self._storage['cluster_id']
fuel_web = self.manager.fuel_web
self.manager.show_step(1)
self.manager.show_step(2)
self.manager.show_step(3)
self.manager.show_step(4)
self.manager.show_step(5)
self.manager.show_step(6)
# assert node disks after deployment
for node_name in self.cluster_config['nodes']:
str_block_devices = fuel_web.get_cluster_block_devices(
node_name)
logger.debug("Block device:\n{}".format(str_block_devices))
expected_regexp = re.compile(
"vda\s+\d+:\d+\s+0\s+{}G\s+0\s+disk".format(
settings.NODE_VOLUME_SIZE))
assert expected_regexp.search(str_block_devices), (
"Unable to find vda block device for {}G in: {}".format(
settings.NODE_VOLUME_SIZE, str_block_devices))
expected_regexp = re.compile(
"vdb\s+\d+:\d+\s+0\s+{}G\s+0\s+disk".format(
settings.NODE_VOLUME_SIZE))
assert expected_regexp.search(str_block_devices), (
"Unable to find vdb block device for {}G in: {}".format(
settings.NODE_VOLUME_SIZE, str_block_devices))
expected_regexp = re.compile(
"vdc\s+\d+:\d+\s+0\s+{}G\s+0\s+disk".format(
settings.NODE_VOLUME_SIZE))
assert expected_regexp.search(str_block_devices), (
"Unable to find vdc block device for {}G in: {}".format(
settings.NODE_VOLUME_SIZE, str_block_devices))
self.manager.show_step(7)
fuel_web.verify_network(cluster_id)
self.manager.show_step(6)
fuel_web.run_ostf(
cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
def nova_service_get_pid(node_ssh, nova_services=None):
pid_dict = {}
for el in nova_services:
cmd = "pgrep {0}".format(el)
pid_dict[el] = node_ssh.execute(cmd)['stdout']
logger.debug('current dict is {0}'. format(pid_dict))
return pid_dict
def separate_db_service_controller_shutdown(self):
"""Shutdown primary controller node
Scenario:
1. Revert snapshot separate_db_service
2. Shutdown primary controller node
3. Wait rabbit and db are operational
4. Run OSTF
Duration 30m
"""
self.env.revert_snapshot("separate_db_service")
cluster_id = self.fuel_web.get_last_created_cluster()
# shutdown primary controller
controller = self.fuel_web.get_nailgun_primary_node(
self.env.d_env.nodes().slaves[0])
logger.debug(
"controller with primary role is {}".format(controller.name))
controller.destroy()
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
controller)['online'], timeout=60 * 5)
self.fuel_web.assert_ha_services_ready(cluster_id)
self.fuel_web.assert_os_services_ready(cluster_id, timeout=15 * 60,
should_fail=1)
self.fuel_web.run_ostf(
cluster_id=cluster_id, should_fail=1)
def replace_centos_repos(repos_attr, upstream_host):
# Walk thru repos_attr and replace/add extra Centos mirrors
if help_data.MIRROR_CENTOS:
logger.debug("Adding new mirrors: '{0}'"
.format(help_data.MIRROR_CENTOS))
repos = add_centos_mirrors()
# Keep other (not upstream) repos, skip previously added ones
for repo_value in repos_attr['value']:
# self.admin_node_ip while repo is located on master node
if upstream_host not in repo_value['uri']:
if check_new_centos_repo(repos, repo_value):
repos.append(repo_value)
else:
logger.debug("Removing mirror: '{0} {1}'"
.format(repo_value['name'], repo_value['uri']))
else:
# Use defaults from Nailgun if MIRROR_CENTOS is not set
repos = repos_attr['value']
if help_data.EXTRA_RPM_REPOS:
repos = add_centos_extra_mirrors(repos=repos)
if help_data.PATCHING_DISABLE_UPDATES:
repos = [repo for repo in repos if repo['name']
not in ('mos-updates', 'mos-security')]
return repos
def generate_facts(ip):
ssh_manager = SSHManager()
facter_dir = '/var/lib/puppet/lib/facter'
exluded_facts = ['naily.rb']
if not ssh_manager.isdir_on_remote(ip, facter_dir):
ssh_manager.mkdir_on_remote(ip, facter_dir)
logger.debug('Directory {0} was created'.format(facter_dir))
ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir))
logger.debug('rb files were removed from {0}'.format(facter_dir))
facts_files = ssh_manager.execute_on_remote(
ip,
'find /etc/puppet/modules/ -wholename "*/lib/facter/*.rb"')['stdout']
facts_files = [i.strip() for i in facts_files]
logger.debug('The following facts {0} will'
' be copied to {1}'.format(facts_files, facter_dir))
for fact in facts_files:
if not fact or re.sub(r'.*/', '', fact) in exluded_facts:
continue
ssh_manager.execute_on_remote(ip,
'cp {0} {1}/'.format(fact, facter_dir))
logger.debug('Facts were copied')
ssh_manager.execute_on_remote(ip, 'facter -p -y > /tmp/facts.yaml')
logger.info('Facts yaml was created')
ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir))
logger.debug('rb files were removed from {0}'.format(facter_dir))
def wrapper(*args, **kwagrs):
try:
return func(*args, **kwagrs)
except SkipTest:
raise SkipTest()
except Exception:
if args and 'snapshot' in args[0].__dict__:
name = 'error_%s' % args[0].snapshot
description = "Failed in method '%s'." % args[0].snapshot
else:
name = 'error_%s' % func.__name__
description = "Failed in method '%s'." % func.__name__
if args[0].env is not None:
try:
create_diagnostic_snapshot(args[0].env,
"fail", name)
except:
logger.error(traceback.format_exc())
raise
finally:
logger.debug(args)
args[0].env.make_snapshot(snapshot_name=name[-50:],
description=description,
is_make=True)
raise
def set_repo_for_perestroika(remote):
"""Set Perestroika repos.
:param remote: Remote node for proceed.
"""
repo = settings.PERESTROIKA_REPO
cmd = "curl {0}".format(repo)
result = remote.execute(cmd)
LOGGER.debug(result)
asserts.assert_equal(result["exit_code"], 0, "Perestroika repos unavailable from node.")
cmd = (
"echo '[mos]\n"
"name=mos\n"
"type=rpm-md\n"
"baseurl={0}\n"
"gpgcheck=0\n"
"enabled=1\n"
"priority=5' >"
"/etc/yum.repos.d/mos.repo && "
"yum clean all".format(repo)
)
result = remote.execute(cmd)
LOGGER.debug(result)
asserts.assert_equal(result["exit_code"], 0, "Can not create config file for repo")
def check_ceph_health(remote, health_status=['HEALTH_OK']):
ceph_health = get_ceph_health(remote)
if all(x in ceph_health.split() for x in health_status):
return True
logger.debug('Ceph health {0} doesn\'t equal to {1}'.format(
ceph_health, ''.join(health_status)))
return False
def _contain_public_ip(data, _used_networks):
_has_puplic_ip = False
_ip_regex = (r'\b((\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])\.){3}'
r'(\d|[1-9]\d|1\d{2}|2[0-4]\d|25[0-5])\b')
_not_public_regex = [
r'\b10(\.\d{1,3}){3}',
r'\b127(\.\d{1,3}){3}',
r'\b169\.254(\.\d{1,3}){2}',
r'172\.(1[6-9]|2[0-9]|3[0-1])(\.\d{1,3}){2}',
r'192\.168(\.\d{1,3}){2}',
r'2(2[4-9]|[3-5][0-9])(\.\d{1,3}){3}'
]
for _match in re.finditer(_ip_regex, data):
# If IP address isn't public and doesn't belong to defined for
# deployment pools (e.g. admin, public, storage), then skip it
if any(re.search(_r, _match.group()) for _r in _not_public_regex) \
and not any(IPAddress(_match.group()) in IPNetwork(net) for
net in _used_networks):
continue
logger.debug('Usage statistics with piblic IP(s):\n {0}'.
format(data))
logger.error('Found public IP in usage statistics: "{0}"'.format(
_match.group()))
_has_puplic_ip = True
return _has_puplic_ip
def check_mcollective_version(self):
"""Check mcollective package version on bootstrap and provisioned node
Scenario:
1. Check mcollective version on bootstrap
2. Create cluster
3. Add one node to cluster
4. Provision nodes
5. Check mcollective version on node
Duration 5m
"""
self.env.revert_snapshot("ready_with_1_slaves", skip_timesync=True)
self.show_step(1)
node = self.env.d_env.get_node(name__in=["slave-01"])
_ip = self.fuel_web.get_nailgun_node_by_devops_node(node)["ip"]
check_package_version(_ip, "mcollective", "2.3.3", "ge")
self.show_step(2)
cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__)
pkg_list = self.fuel_web.get_cluster_ibp_packages(cluster_id)
logger.debug("Cluster IBP packages: {}".format(pkg_list))
self.show_step(3)
self.fuel_web.update_nodes(cluster_id, {"slave-01": ["controller"]})
self.show_step(4)
self.fuel_web.provisioning_cluster_wait(cluster_id)
self.show_step(5)
check_package_version(_ip, "mcollective", "2.3.3", "ge")
def check_ibp_default_package_list(self):
"""Provision one node with default package list
Scenario:
1. Create cluster
2. Add one node to cluster
3. Provision nodes
4. Check that all default packages are installed on the node
Duration 60m
Snapshot check_ibp_default_package_list
"""
self.env.revert_snapshot("ready_with_1_slaves")
self.show_step(1)
cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__)
pkg_list = self.fuel_web.get_cluster_ibp_packages(cluster_id)
logger.debug("Cluster IBP packages: {}".format(pkg_list))
self.show_step(2)
self.fuel_web.update_nodes(cluster_id, {"slave-01": ["controller"]})
self.show_step(3)
self.fuel_web.provisioning_cluster_wait(cluster_id)
self.show_step(4)
self.check_node_packages("slave-01", pkg_list)
self.env.make_snapshot("check_ibp_default_package_list")
def register_rhel_subscription(remote):
"""Register RHEL subscription.
:param remote: Remote node to proceed.
"""
reg_command = (
"/usr/sbin/subscription-manager register "
"--username={0} --password={1}".format(
settings.RHEL_LICENSE_USERNAME,
settings.RHEL_LICENSE_PASSWORD)
)
if settings.RHEL_SERVER_URL:
reg_command = reg_command + " --serverurl={0}".format(
settings.RHEL_SERVER_URL)
if settings.RHEL_REGISTERED_ORG_NAME:
reg_command = reg_command + " --org={0}".format(
settings.RHEL_REGISTERED_ORG_NAME)
if settings.RHEL_RELEASE:
reg_command = reg_command + " --release={0}".format(
settings.RHEL_RELEASE)
if settings.RHEL_ACTIVATION_KEY:
reg_command = reg_command + " --activationkey={0}".format(
settings.RHEL_ACTIVATION_KEY)
cmd = reg_command + " --auto-attach"
result = remote.execute(cmd)
LOGGER.debug(result)
asserts.assert_equal(result['exit_code'], 0, 'RHEL registation failed')
def step_5_run_cluster_haproxy_controller(self):
"""Execute cluster-haproxy task on controller, create snapshot
Depends:
"run_tasks_end_with_vips"
Scenario:
1. Revert snapshot "run_virtual_ips_controller"
2. Get cluster id
3. Execute cluster-haproxy task on controller
4. Verify that task was finished with success.
5. Assert task execution
6. Create snapshot
Snapshot: "step_5_run_cluster_haproxy_controller"
"""
self.check_run_by_group('step_5_run_cluster_haproxy_controller',
'cluster_haproxy_controller')
self.env.revert_snapshot("run_tasks_end_with_vips")
cluster_id = self.get_cluster_id()
controller_id = [
n['id'] for n in
self.fuel_web.client.list_cluster_nodes(cluster_id)
if 'controller' in n['roles']]
self.sync_manifest_to_the_slaves(
cluster_id=cluster_id,
node_ids=controller_id)
tasks = self.fuel_web.client.get_end_deployment_tasks(
cluster_id, end='cluster-haproxy')
pre_cluster_haproxy = self.get_pre_test(tasks, 'cluster-haproxy')
post_cluster_haproxy = self.get_post_test(tasks, 'cluster-haproxy')
if pre_cluster_haproxy:
for node in ['slave-01']:
with self.fuel_web.get_ssh_for_node(node) as ssh:
gd.run_check_from_task(
remote=ssh,
path=pre_cluster_haproxy[0]['cmd'])
res = self.fuel_web.client.put_deployment_tasks_for_cluster(
cluster_id, data=['conntrackd'],
node_id='{0}'.format(controller_id[0]))
logger.debug('res info is {0}'.format(res))
self.fuel_web.assert_task_success(task=res)
res = self.fuel_web.client.put_deployment_tasks_for_cluster(
cluster_id, data=['cluster-haproxy'],
node_id='{0}'.format(controller_id[0]))
logger.debug('res info is {0}'.format(res))
self.fuel_web.assert_task_success(task=res)
if post_cluster_haproxy:
for node in ['slave-01']:
with self.fuel_web.get_ssh_for_node(node) as ssh:
gd.run_check_from_task(
remote=ssh,
path=post_cluster_haproxy[0]['cmd'])
self.env.make_snapshot("step_5_run_cluster_haproxy_controller")
def _create_net_subnet(self, cluster):
"""Create net and subnet"""
contrail_ip = self.fuel_web.get_public_vip(cluster)
logger.info('The ip is %s', contrail_ip)
net = Common(
controller_ip=contrail_ip, user='******',
password='******', tenant='admin'
)
net.neutron.create_network(body={
'network': {
'name': 'net04',
'admin_state_up': True,
}
})
network_id = ''
network_dic = net.neutron.list_networks()
for dd in network_dic['networks']:
if dd.get("name") == "net04":
network_id = dd.get("id")
if network_id == "":
logger.error('Network id empty')
logger.debug("id {0} to master node".format(network_id))
net.neutron.create_subnet(body={
'subnet': {
'network_id': network_id,
'ip_version': 4,
'cidr': '10.100.0.0/24',
'name': 'subnet04',
}
})
def check_ipconfig_for_template(self, cluster_id, network_template,
networks):
logger.info("Checking that IP addresses configuration on nodes "
"corresponds to used networking template...")
# Network for Neutron is configured in namespaces (l3/dhcp agents)
# and a bridge for it doesn't have IP, so skipping it for now
skip_roles = {'neutron/private'}
for node in self.fuel_web.client.list_cluster_nodes(cluster_id):
node_networks = set()
node_group_name = [ng['name'] for ng in
self.fuel_web.client.get_nodegroups()
if ng['id'] == node['group_id']][0]
for role in node['roles']:
node_networks.update(
self.get_template_ep_for_role(template=network_template,
role=role,
nodegroup=node_group_name,
skip_net_roles=skip_roles))
with self.env.d_env.get_ssh_to_remote(node['ip']) as remote:
for network in networks:
if network['name'] not in node_networks or \
network['group_id'] != node['group_id']:
continue
logger.debug('Checking interface "{0}" for IP network '
'"{1}" on "{2}"'.format(network['interface'],
network['cidr'],
node['hostname']))
self.check_interface_ip_exists(remote,
network['interface'],
network['cidr'])
def wrapper(*args, **kwagrs):
try:
return func(*args, **kwagrs)
except SkipTest:
raise SkipTest()
except Exception as test_exception:
exc_trace = sys.exc_traceback
if args and 'snapshot' in args[0].__dict__:
name = 'error_%s' % args[0].snapshot
description = "Failed in method '%s'." % args[0].snapshot
else:
name = 'error_%s' % func.__name__
description = "Failed in method '%s'." % func.__name__
if args[0].env is not None:
try:
create_diagnostic_snapshot(args[0].env,
"fail", name)
except:
logger.error("Fetching of diagnostic snapshot failed: {0}".
format(traceback.format_exc()))
try:
admin_remote = args[0].env.get_admin_remote()
pull_out_logs_via_ssh(admin_remote, name)
except:
logger.error("Fetching of raw logs failed: {0}".
format(traceback.format_exc()))
finally:
logger.debug(args)
args[0].env.make_snapshot(snapshot_name=name[-50:],
description=description,
is_make=True)
raise test_exception, None, exc_trace
def replace_package(remote, package_name, package_path):
cmd = "ls -all {0} | grep noarch.rpm| awk '{{print $9}}' ".format(
package_path)
result = remote.execute(cmd)
asserts.assert_equal(
0, result['exit_code'],
'Failed to run command {0} with {1} '
'on replace package stage'.format(cmd, result))
package_from_review = ''.join(result['stdout']).strip()
income_version = get_package_version(
remote, os.path.join(package_path, package_from_review),
income=True)
logger.info('Version of package from review {0}'.format(
income_version))
installed_rpm = get_package_version(
remote, package_name)
logger.info('Version of installed package{0}'.format(installed_rpm))
if installed_rpm != income_version:
logger.info('Try to install package {0}'.format(
package_from_review))
cmd = 'rpm -Uvh --oldpackage {0}{1}*.noarch.rpm'.format(
package_path, package_name)
install_result = remote.execute(cmd)
logger.debug('Install package result {0}'.format(install_result))
installed_rpm = get_package_version(
remote, package_name)
asserts.assert_equal(
installed_rpm, income_version,
'Package {0} from review '
'installation fails. Current installed '
'package is {1}'.format(income_version, installed_rpm))
def __init__(self):
logger.debug('SSH_MANAGER: Run constructor SSHManager')
self.__connections = {} # Disallow direct type change and deletion
self.admin_ip = None
self.admin_port = None
self.login = None
self.__password = None
def upload_tarball(node_ssh, tar_path, tar_target):
try:
logger.debug("Start to upload tar file")
node_ssh.upload(tar_path, tar_target)
except Exception:
logger.error('Failed to upload file')
logger.error(traceback.format_exc())
def replace_ubuntu_repos(repos_attr, upstream_host):
# Walk thru repos_attr and replace/add extra Ubuntu mirrors
repos = []
if help_data.MIRROR_UBUNTU:
logger.debug("Adding new mirrors: '{0}'"
.format(help_data.MIRROR_UBUNTU))
repos = add_ubuntu_mirrors()
# Keep other (not upstream) repos, skip previously added ones
for repo_value in repos_attr['value']:
if upstream_host not in repo_value['uri']:
if check_new_ubuntu_repo(repos, repo_value):
repos.append(repo_value)
else:
logger.debug("Removing mirror: '{0} {1}'"
.format(repo_value['name'], repo_value['uri']))
else:
# Use defaults from Nailgun if MIRROR_UBUNTU is not set
repos = repos_attr['value']
if help_data.EXTRA_DEB_REPOS:
repos = add_ubuntu_extra_mirrors(repos=repos)
if help_data.PATCHING_DISABLE_UPDATES:
for repo in repos:
if repo['name'] in ('mos-updates', 'mos-security'):
repos.remove(repo)
return repos
def _verify():
ret = remote.check_call('/usr/bin/nova-manage service list')
logger.debug("Service list: {}".format(ret['stdout']))
assert_equal(
smiles_count, ''.join(ret['stdout']).count(":-)"), "Smiles count")
assert_equal(
0, ''.join(ret['stdout']).count("XXX"), "Broken services count")
def get_sha_sum(file_path):
logger.debug('Get md5 fo file {0}'.format(file_path))
md5_sum = SSHManager().execute_on_remote(
SSHManager().admin_ip, cmd='md5sum {0}'.format(
file_path))['stdout_str'].strip()
logger.info('MD5 is {0}'.format(md5_sum))
return md5_sum
def replace_rpm_package(package):
"""Replaced rpm package.rpm on master node with package.rpm
from review
"""
ssh = SSHManager()
logger.info("Patching {}".format(package))
if not settings.UPDATE_FUEL:
raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True')
try:
# Upload package
target_path = '/var/www/nailgun/{}/'.format(package)
ssh.upload_to_remote(
ip=ssh.admin_ip,
source=settings.UPDATE_FUEL_PATH.rstrip('/'),
target=target_path)
package_name = package
package_ext = '*.noarch.rpm'
pkg_path = os.path.join(target_path,
'{}{}'.format(package_name, package_ext))
full_package_name = get_full_filename(wildcard_name=pkg_path)
logger.debug('Package name is {0}'.format(full_package_name))
full_package_path = os.path.join(os.path.dirname(pkg_path),
full_package_name)
# Update package on master node
if not does_new_pkg_equal_to_installed_pkg(
installed_package=package_name,
new_package=full_package_path):
update_rpm(path=full_package_path)
except Exception:
logger.error("Could not upload package")
raise
def separate_all_service_controller_shutdown(self):
"""Shutdown primary controller node
Scenario:
1. Revert snapshot separate_all_service
2. Shutdown primary controller node
3. Wait HA is working
4. Run OSTF
Duration 30m
Snapshot None
"""
self.env.revert_snapshot("separate_all_service")
cluster_id = self.fuel_web.get_last_created_cluster()
# shutdown primary controller
controller = self.fuel_web.get_nailgun_primary_node(
self.env.d_env.nodes().slaves[0])
logger.debug(
"controller with primary role is {}".format(controller.name))
controller.destroy()
self.fuel_web.wait_node_is_offline(controller)
# One test should fail: Check state of haproxy backends on controllers
self.fuel_web.assert_ha_services_ready(cluster_id, should_fail=1)
self.fuel_web.assert_os_services_ready(cluster_id, timeout=15 * 60)
self.fuel_web.run_ostf(cluster_id=cluster_id)
def wrapper(*args, **kwargs):
result = func(*args, **kwargs)
try:
cluster_id = args[0].env.fuel_web.get_last_created_cluster()
logger.info("start checking snapshot logs")
controllers = \
args[0].env.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['controller'])
computes = \
args[0].env.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['compute'])
logger.debug("controller nodes are {}".format(controllers))
logger.debug("compute nodes are {}".format(computes))
controllers_fqdns = [controller['fqdn']
for controller in controllers]
compute_fqdns = [compute['fqdn'] for compute in computes]
logger.debug("controller fqdns are {}".format(controllers_fqdns))
logger.debug("compute fqdns are {}".format(compute_fqdns))
args[0].env.fuel_web.task_wait(
args[0].env.fuel_web.client.generate_logs(), 60 * 10)
snapshot_name = args[0].env.ssh_manager.execute_on_remote(
args[0].env.ssh_manager.admin_ip,
cmd="ls -I *.tar.xz /var/dump/")['stdout_str']
logger.debug("snapshot name is {}".format(snapshot_name))
check_snapshot_logs(args[0].env.ssh_manager.admin_ip,
snapshot_name, controllers_fqdns,
compute_fqdns)
return result
except Exception:
logger.error(traceback.format_exc())
raise
def ovs_get_tag_by_port(remote, port):
"""Get the tag used for OVS interface by Neutron port ID
:param SSHClient remote: fuel-devops.helpers.helpers object
:param str port: Neutron port ID
:return str: tag number
"""
interfaces_raw = ovs_get_data(remote,
table='Interface',
columns=['external_ids', 'name'])
interfaces = ovs_decode_columns(interfaces_raw)
ports_ifaces = {x['external_ids']['iface-id']: x['name']
for x in interfaces if 'iface-id' in x['external_ids']}
logger.debug("OVS interfaces: {0}".format(ports_ifaces))
if port not in ports_ifaces:
raise ValueError("Neutron port {0} not found in OVS interfaces."
.format(port))
iface_id = ports_ifaces[port]
ovs_port_raw = ovs_get_data(remote,
table='Port {0}'.format(iface_id),
columns=['tag'])
ovs_port = ovs_decode_columns(ovs_port_raw)
logger.debug("OVS tag for port {0}: {1}".format(iface_id, ovs_port))
ovs_tag = ovs_port[0]['tag']
return str(ovs_tag)
def ha_delete_vips(self):
if not self.env.d_env.has_snapshot(self.snapshot_name):
raise SkipTest()
logger.debug('Start reverting of {0} snapshot'.format(
self.snapshot_name))
self.env.revert_snapshot(self.snapshot_name)
cluster_id = \
self.fuel_web.client.get_cluster_id(self.__class__.__name__)
logger.debug('Cluster id is {0}'.format(cluster_id))
resources = {
"vip__management": {
"iface": "b_management",
"netns": "haproxy"
},
"vip__public": {
"iface": "b_public",
"netns": "haproxy"
}
}
nailgun_controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id=cluster_id, roles=['controller'])
devops_controllers = self.fuel_web.get_devops_nodes_by_nailgun_nodes(
nailgun_controllers)
assert_true(devops_controllers is not None,
"Nailgun nodes don't associating to devops nodes")
logger.debug("Current controller nodes are {0}".format(
[i.name for i in devops_controllers]))
for resource in resources:
for check_counter in xrange(1, 11):
# 1. Locate where resource is running
active_nodes = self.fuel_web.get_pacemaker_resource_location(
devops_controllers[0].name, resource)
assert_true(
len(active_nodes) == 1,
"Resource should be running on a single node, "
"but started on the nodes {0}".format(
[n.name for n in active_nodes]))
logger.debug("Start looking for the IP of {0} "
"on {1}".format(resource, active_nodes[0].name))
address = self.fuel_web.ip_address_show(
active_nodes[0].name,
interface=resources[resource]['iface'],
namespace=resources[resource]['netns'])
assert_true(
address is not None,
"Resource {0} located on {1}, but interface "
"doesn't have "
"ip address".format(resource, active_nodes[0].name))
logger.debug("Found the IP: {0}".format(address))
# 2. Deleting VIP
logger.debug("Start ip {0} deletion on node {1} and "
"interface {2} ".format(address,
active_nodes[0].name,
resources[resource]))
self.fuel_web.ip_address_del(
node_name=active_nodes[0].name,
interface=resources[resource]['iface'],
ip=address,
namespace=resources[resource]['netns'])
def check_restore():
new_nodes = self.fuel_web.get_pacemaker_resource_location(
devops_controllers[0].name, resource)
if len(new_nodes) != 1:
return False
new_address = self.fuel_web.ip_address_show(
new_nodes[0].name,
interface=resources[resource]['iface'],
namespace=resources[resource]['netns'])
if new_address is None:
return False
else:
return True
# 3. Waiting for restore the IP
logger.debug("Waiting while deleted ip restores ...")
try:
wait(check_restore, timeout=60)
except TimeoutError as e:
logger.error("Resource has not been restored for a 60 sec")
raise e
new_nodes = self.fuel_web.get_pacemaker_resource_location(
devops_controllers[0].name, resource)
assert_true(
len(new_nodes) == 1,
"After ip deletion resource should run on a single"
" node, but runned on {0}. On {1} attempt".format(
[n.name for n in new_nodes], check_counter))
logger.info("Resource has been deleted from {0} and "
"restored on {1}".format(active_nodes[0].name,
new_nodes[0].name))
logger.info("Resource {0} restored "
"{1} times".format(resource, check_counter))
# Run OSTF tests
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
def test_log_rotation(self):
"""Logrotate with logrotate.conf on master node
Scenario:
1. Revert snapshot "empty"
2. Check free disk space under /var/log, check free inodes
3. Generate 2GB size file
4. Run logrotate 2 times
5. Check free disk space, check free inodes
Duration 30m
"""
self.env.revert_snapshot("empty")
with self.env.d_env.get_admin_remote() as remote:
# get data before logrotate
free, suff = self.check_free_space(remote)
free_inodes, i_suff = self.check_free_inodes(remote)
logger.debug('Free inodes before file '
'creation: {0}{1}'.format(free_inodes, i_suff))
self.generate_file(remote,
size='2G',
path='/var/log/',
name='messages')
free2, suff2 = self.check_free_space(remote)
assert_true(
free2 < free, 'File was not created. Free space '
'before creation {0}{1}, '
'free space after '
'creation {2}{3}'.format(free, suff, free2, suff2))
self.execute_logrotate_cmd(remote)
free3, suff3 = self.check_free_space(remote)
res = self.execute_logrotate_cmd(remote, exit_code=1)
# Expect 1 exit code here, according
# to some rotated logs are skipped to rotate
# second run. That's caused 1
assert_equal(1, res['exit_code'])
assert_equal(
False, 'error' in res['stderr'],
'Second run of logrotate failed'
' with {0}'.format(res['stderr']))
free4, suff4 = self.check_free_space(remote)
free_inodes4, i_suff4 = self.check_free_inodes(remote)
logger.info('Free inodes after logrotation:'
' {0}{1}'.format(free_inodes4, i_suff4))
assert_true(
free4 > free3, 'Logs were not rotated. '
'Rotate was executed 2 times. '
'Free space after first rotation: {0}{1}, '
'after second {2}{3} free space before rotation {4}'
'{5}'.format(free3, suff3, free4, suff4, free, suff))
assert_equal(
(free_inodes, i_suff), (free_inodes4, i_suff4),
'Unexpected free inodes count. Before log rotate was: {0}{1}'
' after logrotation: {2}{3}'.format(free_inodes, i_suff,
free_inodes4, i_suff4))
self.env.make_snapshot("test_logrotate")
def shutdown_ceph_for_all(self):
"""Shutdown of Neutron Vxlan, ceph for all cluster
Scenario:
1. Create cluster with Neutron Vxlan, ceph for all,
ceph replication factor - 3
2. Add 3 controller, 2 compute, 3 ceph nodes
3. Verify Network
4. Deploy cluster
5. Verify networks
6. Run OSTF
7. Create 2 volumes and 2 instances with attached volumes
8. Fill ceph storages up to 30%(15% for each instance)
9. Shutdown of all nodes
10. Wait 5 minutes
11. Start cluster
12. Wait until OSTF 'HA' suite passes
13. Verify networks
14. Run OSTF tests
Duration 230m
"""
self.env.revert_snapshot('ready_with_9_slaves')
self.show_step(1, initialize=True)
data = {
'tenant': 'failover',
'user': '******',
'password': '******',
"net_provider": 'neutron',
"net_segment_type": settings.NEUTRON_SEGMENT['tun'],
'volumes_ceph': True,
'images_ceph': True,
'ephemeral_ceph': True,
'objects_ceph': True,
'osd_pool_size': '3',
'volumes_lvm': False,
}
cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__,
settings=data)
self.show_step(2)
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute'],
'slave-05': ['compute'],
'slave-06': ['ceph-osd'],
'slave-07': ['ceph-osd'],
'slave-08': ['ceph-osd']
})
self.show_step(3)
self.fuel_web.verify_network(cluster_id)
self.show_step(4)
self.fuel_web.deploy_cluster_wait(cluster_id)
self.show_step(5)
self.fuel_web.verify_network(cluster_id)
self.show_step(6)
self.fuel_web.run_ostf(cluster_id)
self.show_step(7)
os = os_actions.OpenStackActions(
controller_ip=self.fuel_web.get_public_vip(cluster_id),
user='******',
passwd='failover',
tenant='failover')
net_name = self.fuel_web.get_cluster_predefined_networks_name(
cluster_id)['private_net']
hypervisors = os.get_hypervisors()
hypervisor_name = hypervisors[0].hypervisor_hostname
instance_1 = os.create_server_for_migration(
neutron=True,
availability_zone="nova:{0}".format(hypervisor_name),
label=net_name)
logger.info("New instance {0} created on {1}".format(
instance_1.id, hypervisor_name))
floating_ip_1 = os.assign_floating_ip(instance_1)
logger.info("Floating address {0} associated with instance {1}".format(
floating_ip_1.ip, instance_1.id))
hypervisor_name = hypervisors[1].hypervisor_hostname
instance_2 = os.create_server_for_migration(
neutron=True,
availability_zone="nova:{0}".format(hypervisor_name),
label=net_name)
logger.info("New instance {0} created on {1}".format(
instance_2.id, hypervisor_name))
floating_ip_2 = os.assign_floating_ip(instance_2)
logger.info("Floating address {0} associated with instance {1}".format(
floating_ip_2.ip, instance_2.id))
self.show_step(8)
ceph_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['ceph-osd'])
total_ceph_size = 0
for node in ceph_nodes:
total_ceph_size += \
self.fuel_web.get_node_partition_size(node['id'], 'ceph')
percent_15_mb = 0.15 * total_ceph_size
percent_15_gb = percent_15_mb // 1024
volume_size = int(percent_15_gb + 1)
volume_1 = os.create_volume(size=volume_size)
volume_2 = os.create_volume(size=volume_size)
logger.info('Created volumes: {0}, {1}'.format(volume_1.id,
volume_2.id))
ip = self.fuel_web.get_nailgun_node_by_name("slave-01")['ip']
logger.info("Attach volumes")
cmd = 'nova volume-attach {srv_id} {volume_id} /dev/vdb'
self.ssh_manager.execute_on_remote(
ip=ip,
cmd='. openrc; ' +
cmd.format(srv_id=instance_1.id, volume_id=volume_1.id))
self.ssh_manager.execute_on_remote(
ip=ip,
cmd='. openrc; ' +
cmd.format(srv_id=instance_2.id, volume_id=volume_2.id))
cmds = [
'sudo sh -c "/usr/sbin/mkfs.ext4 /dev/vdb"',
'sudo sh -c "/bin/mount /dev/vdb /mnt"',
'sudo sh -c "/usr/bin/nohup'
' /bin/dd if=/dev/zero of=/mnt/bigfile '
'bs=1M count={} &"'.format(int(percent_15_mb))
]
md5s = {floating_ip_1.ip: '', floating_ip_2.ip: ''}
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
for ip in [floating_ip_1.ip, floating_ip_2.ip]:
for cmd in cmds:
res = os.execute_through_host(remote, ip, cmd)
logger.info('RESULT for {}: {}'.format(
cmd, utils.pretty_log(res)))
logger.info('Wait 7200 untill "dd" ends')
for _ in range(720):
cmd = 'ps -ef |grep -v grep| grep "dd if" '
res = os.execute_through_host(remote, ip, cmd)
if res['exit_code'] != 0:
break
time.sleep(10)
logger.debug('Wait another 10 sec -'
' totally waited {} sec'.format(10 * _))
else:
raise TimeoutError('BigFile has not been'
' created yet, after 7200 sec')
cmd = 'md5sum /mnt/bigfile'
md5s[ip] = os.execute_through_host(remote, ip, cmd)['stdout']
self.show_step(9)
nodes = {'compute': [], 'controller': [], 'ceph-osd': []}
for role in nodes:
nailgun_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, [role])
nodes[role] = self.fuel_web.get_devops_nodes_by_nailgun_nodes(
nailgun_nodes)
self.fuel_web.warm_shutdown_nodes(nodes['compute'])
self.fuel_web.warm_shutdown_nodes(nodes['controller'])
self.fuel_web.warm_shutdown_nodes(nodes['ceph-osd'])
self.show_step(10)
time.sleep(300)
self.show_step(11)
self.fuel_web.warm_start_nodes(nodes['ceph-osd'])
self.fuel_web.warm_start_nodes(nodes['controller'])
self.show_step(12)
self.fuel_web.assert_ha_services_ready(cluster_id)
self.fuel_web.warm_start_nodes(nodes['compute'])
self.fuel_web.assert_os_services_ready(cluster_id)
self.show_step(13)
self.fuel_web.verify_network(cluster_id)
self.show_step(14)
self.fuel_web.run_ostf(cluster_id)
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
for ip in [floating_ip_1.ip, floating_ip_2.ip]:
cmd = 'md5sum /mnt/bigfile'
md5 = os.execute_through_host(remote, ip, cmd)['stdout']
assert_equal(
md5, md5s[ip], "Actual md5sum {0} doesnt match"
" with old one {1} on {2}".format(md5, md5s[ip], ip))
def slave_nodes_compliance(self):
""" Install and run lynis on slave nodes
Scenario:
1. Revert snapshot ready_with_3_slaves
2. Create cluster with 3 nodes: controller, compute, cinder
3 Install lynis package on slaves
4. Run lynis custom test
5. Analyze lynis results
Duration: 30 min
"""
self.show_step(1)
self.env.revert_snapshot('ready_with_3_slaves')
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
)
self.show_step(2)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller'],
'slave-02': ['compute'],
'slave-03': ['cinder']
}
)
self.fuel_web.deploy_cluster_wait(cluster_id)
self.show_step(3)
cmd =\
'echo 172.18.162.63 perestroika-repo-tst.infra.mirantis.net' \
' >> /etc/hosts'
for node in self.fuel_web.client.list_cluster_nodes(cluster_id):
self.ssh_manager.execute_on_remote(node['ip'], cmd)
cmd =\
'sudo add-apt-repository "http://perestroika-repo-tst.' \
'infra.mirantis.net/mos-packaging/ubuntu/"'
for node in self.fuel_web.client.list_cluster_nodes(cluster_id):
self.ssh_manager.execute_on_remote(node['ip'], cmd)
cmd = 'apt-get install lynis'
for node in self.fuel_web.client.list_cluster_nodes(cluster_id):
self.ssh_manager.execute_on_remote(node['ip'], cmd)
# check that lynis version is correct and installed from perestroika
nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
self.show_step(4)
cmd = 'lynis -c -Q --tests-category "custom"'
for node in nodes:
self.ssh_manager.execute_on_remote(node['ip'], cmd)
self.show_step(5)
cmd = \
'awk -F\']\' \'/Mirantis\S+\sResult/ {print $2}\' ' \
'/var/log/lynis.log'
for node in nodes:
lynis_failed_tests = [
test for test in self.ssh_manager.execute_on_remote(
node['ip'], cmd)['stdout']]
logger.debug(lynis_failed_tests)
self.show_step(4)
# Check that lynis test haven't failed tests
assert_equal(len(lynis_failed_tests),
0,
message="Some lynis tests was failed for node {}."
"Please check lynis logs".format(node['ip']))
def gate_patch_astute(self):
""" Revert 'ready_with_3_slaves' snapshot,
download package with astute changes, install package,
start deployment, check for regression
Scenario:
1. Revert environment ready_with_3_slaves
2. Upload package
3. Update Astute rpm package from review
4. Update network configuration
5. Create env
6. Update nodes with roles: controller, compute, cinder
7. Deploy cluster
8. Start ostf tests to check that changes do not reproduce regression
"""
if not settings.UPDATE_FUEL:
raise Exception('UPDATE_FUEL variable is not set. '
'UPDATE_FUEL value is {}'.format(
settings.UPDATE_FUEL))
astute_service = 'astute'
package_name = 'rubygem-astute'
package_ext = '*.noarch.rpm'
target_path = '/var/www/nailgun/astute/'
self.show_step(1)
self.env.revert_snapshot('ready_with_3_slaves')
self.show_step(2)
self.ssh_manager.upload_to_remote(
self.ssh_manager.admin_ip,
source=settings.UPDATE_FUEL_PATH.rstrip('/'),
target=target_path)
self.show_step(3)
pkg_path = os.path.join(target_path,
'{}{}'.format(package_name, package_ext))
logger.debug('Package path is {0}'.format(pkg_path))
full_package_name = utils.get_full_filename(wildcard_name=pkg_path)
logger.debug('Package name is {0}'.format(full_package_name))
full_package_path = os.path.join(os.path.dirname(pkg_path),
full_package_name)
if not utils.does_new_pkg_equal_to_installed_pkg(
installed_package=package_name, new_package=full_package_path):
utils.update_rpm(path=full_package_path)
utils.restart_service(service_name=astute_service, timeout=10)
self.show_step(4)
self.fuel_web.change_default_network_settings()
self.show_step(5)
release_id = self.fuel_web.get_releases_list_for_os(
release_name=OPENSTACK_RELEASE)[0]
cmd = ('fuel env create --name={0} --release={1} '
'--nst=tun --json'.format(self.__class__.__name__, release_id))
env_result = self.ssh_manager.execute_on_remote(
self.ssh_manager.admin_ip, cmd=cmd, jsonify=True)['stdout_json']
self.show_step(6)
cluster_id = env_result['id']
logger.debug('cluster id is {0}'.format(cluster_id))
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': ['controller'],
'slave-02': ['compute'],
'slave-03': ['cinder']
})
self.show_step(7)
cluster_id = self.fuel_web.get_last_created_cluster()
self.fuel_web.deploy_cluster_wait(cluster_id)
self.show_step(8)
self.fuel_web.run_ostf(cluster_id=cluster_id, test_sets=['smoke'])
def check_nodes_disks(self):
"""Verify nailgun notifications for discovered nodes
Scenario:
1. Create cluster
2. Add 1 node with controller role
3. Add 1 node with compute role
4. Add 1 node with cinder role
5. Deploy the cluster
6. Verify hard drive sizes for deployed nodes
"""
self.env.revert_snapshot("ready_with_3_slaves")
cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__,
mode=DEPLOYMENT_MODE_SIMPLE)
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': ['controller'],
'slave-02': ['compute'],
'slave-03': ['cinder']
})
self.fuel_web.deploy_cluster_wait(cluster_id)
os_conn = os_actions.OpenStackActions(
self.fuel_web.get_nailgun_node_by_name('slave-01')['ip'])
self.fuel_web.assert_cluster_ready(os_conn,
smiles_count=6,
networks_count=1,
timeout=300)
self.fuel_web.security.verify_firewall(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id)
nodes_dict = {
'slave-01': ['controller'],
'slave-02': ['compute'],
'slave-03': ['cinder']
}
# assert node disks after deployment
for node_name in nodes_dict:
str_block_devices = self.fuel_web.get_cluster_block_devices(
node_name)
logger.debug("Block device:\n{}".format(str_block_devices))
expected_regexp = re.compile(
"vda\s+\d+:\d+\s+0\s+{}G\s+0\s+disk".format(NODE_VOLUME_SIZE))
assert_true(
expected_regexp.search(str_block_devices),
"Unable to find vda block device for {}G in: {}".format(
NODE_VOLUME_SIZE, str_block_devices))
expected_regexp = re.compile(
"vdb\s+\d+:\d+\s+0\s+{}G\s+0\s+disk".format(NODE_VOLUME_SIZE))
assert_true(
expected_regexp.search(str_block_devices),
"Unable to find vdb block device for {}G in: {}".format(
NODE_VOLUME_SIZE, str_block_devices))
expected_regexp = re.compile(
"vdc\s+\d+:\d+\s+0\s+{}G\s+0\s+disk".format(NODE_VOLUME_SIZE))
assert_true(
expected_regexp.search(str_block_devices),
"Unable to find vdc block device for {}G in: {}".format(
NODE_VOLUME_SIZE, str_block_devices))
def check_dead_rabbit_node_kicked(self):
if not self.env.d_env.has_snapshot(self.snapshot_name):
raise SkipTest()
self.env.revert_snapshot(self.snapshot_name)
pcm_nodes = self.fuel_web.get_pcm_nodes(
self.env.d_env.nodes().slaves[0].name, pure=True)['Online']
logger.debug("pcm nodes are {}".format(pcm_nodes))
rabbit_nodes = [node.replace(DNS_SUFFIX, "") for node in pcm_nodes]
logger.debug("rabbit nodes are {}".format(rabbit_nodes))
with self.fuel_web.get_ssh_for_node(
self.env.d_env.nodes().slaves[0].name) as remote:
slave1_name = ''.join(remote.execute('hostname')['stdout']).strip()
logger.debug('slave1 name is {}'.format(slave1_name))
for rabbit_node in rabbit_nodes:
if rabbit_node in slave1_name:
rabbit_slave1_name = rabbit_node
logger.debug("rabbit node is {}".format(rabbit_slave1_name))
pcm_nodes.remove(slave1_name)
with self.fuel_web.get_ssh_for_node(
self.env.d_env.nodes().slaves[0].name) as remote:
remote.execute('crm configure property maintenance-mode=true')
remote.execute('rabbitmqctl stop_app')
remote.execute('service corosync stop')
with self.env.d_env.get_admin_remote() as remote:
cmd = "grep 'Forgetting cluster node rabbit@{0}' /var/log/remote" \
"/{1}/rabbit-fence.log".format(rabbit_slave1_name,
pcm_nodes[0])
try:
wait(lambda: not remote.execute(cmd)['exit_code'],
timeout=2 * 60)
except TimeoutError:
result = remote.execute(cmd)
assert_equal(
0, result['exit_code'], 'dead rabbit node was not removed,'
' result is {}'.format(result))
assert_equal(
0,
remote.execute(
"grep 'Got {0} that left cluster' /var/log/remote/{1}/"
"rabbit-fence.log".format(slave1_name,
pcm_nodes[0]))['exit_code'],
"node {} didn't leave cluster".format(slave1_name))
assert_equal(
0,
remote.execute(
"grep 'Preparing to fence node rabbit@{0} from rabbit cluster'"
" /var/log/remote/{1}/"
"rabbit-fence.log".format(rabbit_slave1_name,
pcm_nodes[0]))['exit_code'],
"node {} wasn't prepared for"
" fencing".format(rabbit_slave1_name))
assert_equal(
0,
remote.execute(
"grep 'Disconnecting node rabbit@{0}' /var/log/remote/{1}/"
"rabbit-fence.log".format(rabbit_slave1_name,
pcm_nodes[0]))['exit_code'],
"node {} wasn't disconnected".format(rabbit_slave1_name))
rabbit_nodes.remove(rabbit_slave1_name)
rabbit_status = self.fuel_web.get_rabbit_running_nodes(
self.env.d_env.nodes().slaves[1].name)
logger.debug("rabbit status is {}".format(rabbit_status))
for rabbit_node in rabbit_nodes:
assert_true(
rabbit_node in rabbit_status, "rabbit node {} is not in"
" rabbit status".format(rabbit_node))
assert_false(
rabbit_slave1_name in rabbit_status, "rabbit node {0} is still in"
" cluster".format(rabbit_slave1_name))
def block_net_traffic_cinder(self):
"""Block network traffic of whole environment
Scenario:
1. Revert environment deploy_ha_cinder
2. Create 2 volumes and 2 instances with attached volumes
3. Fill cinder storages up to 30%
4. Start Rally
5. Block traffic of all networks
6. Sleep 5 minutes
7. Unblock traffic of all networks
8. Wait until cluster nodes become online
9. Verify networks
10. Run OSTF tests
Duration: 40 min
Snapshot: block_net_traffic
"""
self.show_step(1)
self.env.revert_snapshot('deploy_ha_cinder')
cluster_id = self.fuel_web.get_last_created_cluster()
self.show_step(2)
os = os_actions.OpenStackActions(
controller_ip=self.fuel_web.get_public_vip(cluster_id),
user='******',
passwd='failover',
tenant='failover')
net_name = self.fuel_web.get_cluster_predefined_networks_name(
cluster_id)['private_net']
hypervisors = os.get_hypervisors()
hypervisor_name = hypervisors[0].hypervisor_hostname
instance_1 = os.create_server_for_migration(
neutron=True,
availability_zone="nova:{0}".format(hypervisor_name),
label=net_name)
logger.info("New instance {0} created on {1}".format(
instance_1.id, hypervisor_name))
floating_ip_1 = os.assign_floating_ip(instance_1)
logger.info("Floating address {0} associated with instance {1}".format(
floating_ip_1.ip, instance_1.id))
hypervisor_name = hypervisors[1].hypervisor_hostname
instance_2 = os.create_server_for_migration(
neutron=True,
availability_zone="nova:{0}".format(hypervisor_name),
label=net_name)
logger.info("New instance {0} created on {1}".format(
instance_2.id, hypervisor_name))
floating_ip_2 = os.assign_floating_ip(instance_2)
logger.info("Floating address {0} associated with instance {1}".format(
floating_ip_2.ip, instance_2.id))
self.show_step(3)
cinder_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['cinder'])
total_cinder_size = 0
for node in cinder_nodes:
total_cinder_size += \
self.fuel_web.get_node_partition_size(node['id'], 'cinder')
percent_15_mb = 0.15 * total_cinder_size
percent_15_gb = percent_15_mb // 1024
volume_size = int(percent_15_gb + 1)
volume_1 = os.create_volume(size=volume_size)
volume_2 = os.create_volume(size=volume_size)
logger.info('Created volumes: {0}, {1}'.format(volume_1.id,
volume_2.id))
ip = self.fuel_web.get_nailgun_node_by_name("slave-01")['ip']
logger.info("Attach volumes")
cmd = 'nova volume-attach {srv_id} {volume_id} /dev/vdb'
self.ssh_manager.execute_on_remote(
ip=ip,
cmd='. openrc; ' +
cmd.format(srv_id=instance_1.id, volume_id=volume_1.id))
self.ssh_manager.execute_on_remote(
ip=ip,
cmd='. openrc; ' +
cmd.format(srv_id=instance_2.id, volume_id=volume_2.id))
cmds = [
'sudo sh -c "/usr/sbin/mkfs.ext4 /dev/vdb"',
'sudo sh -c "/bin/mount /dev/vdb /mnt"',
'sudo sh -c "/usr/bin/nohup'
' /bin/dd if=/dev/zero of=/mnt/bigfile '
'bs=1M count={} &"'.format(int(percent_15_mb))
]
md5s = {floating_ip_1.ip: '', floating_ip_2.ip: ''}
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
for ip in [floating_ip_1.ip, floating_ip_2.ip]:
for cmd in cmds:
res = os.execute_through_host(remote, ip, cmd)
logger.info('RESULT for {}: {}'.format(
cmd, utils.pretty_log(res)))
logger.info('Wait 7200 untill "dd" ends')
for _ in range(720):
cmd = 'ps -ef |grep -v grep| grep "dd if" '
res = os.execute_through_host(remote, ip, cmd)
if res['exit_code'] != 0:
break
time.sleep(15)
logger.debug('Wait another 15 sec -'
' totally waited {} sec'.format(10 * _))
else:
raise TimeoutError('BigFile has not been'
' created yet, after 7200 sec')
cmd = 'md5sum /mnt/bigfile'
md5s[ip] = os.execute_through_host(remote, ip, cmd)['stdout']
self.show_step(4)
assert_true(settings.PATCHING_RUN_RALLY,
'PATCHING_RUN_RALLY was not set in true')
rally_benchmarks = {}
benchmark_results = {}
for tag in set(settings.RALLY_TAGS):
rally_benchmarks[tag] = RallyBenchmarkTest(
container_repo=settings.RALLY_DOCKER_REPO,
environment=self.env,
cluster_id=cluster_id,
test_type=tag)
benchmark_results[tag] = rally_benchmarks[tag].run()
logger.debug(benchmark_results[tag].show())
self.show_step(5)
nodes = [
node for node in self.env.d_env.get_nodes()
if node.driver.node_active(node)
]
for interface in nodes[1].interfaces:
if interface.is_blocked:
raise Exception('Interface {0} is blocked'.format(interface))
else:
interface.network.block()
self.show_step(6)
time.sleep(60 * 5)
self.show_step(7)
for interface in nodes[1].interfaces:
if interface.network.is_blocked:
interface.network.unblock()
else:
raise Exception(
'Interface {0} was not blocked'.format(interface))
self.show_step(8)
self.fuel_web.wait_nodes_get_online_state(nodes[1:])
self.show_step(9)
self.fuel_web.verify_network(cluster_id)
self.show_step(10)
try:
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
except AssertionError:
time.sleep(600)
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
def negative_auto_cic_maintenance_mode(self):
"""Check negative scenario for auto maintenance mode
Scenario:
1. Revert snapshot
2. Disable UMM
3. Change UMM.CONF
4. Unexpected reboot
5. Check the controller not switching in maintenance mode
6. Check the controller become available
Duration 85m
"""
self.env.revert_snapshot('cic_maintenance_mode')
cluster_id = self.fuel_web.get_last_created_cluster()
# Select a non-primary controller
regular_ctrl = self.fuel_web.get_nailgun_node_by_name("slave-02")
dregular_ctrl = self.fuel_web.get_devops_node_by_nailgun_node(
regular_ctrl)
_ip = regular_ctrl['ip']
_id = regular_ctrl['id']
asserts.assert_true('True' in check_available_mode(_ip),
"Maintenance mode is not available")
logger.info('Disable UMM on node-{0}'.format(_id))
change_config(_ip, umm=False, reboot_count=0)
asserts.assert_false('True' in check_available_mode(_ip),
"Maintenance mode should not be available")
command = 'reboot --force >/dev/null & '
logger.info('Unexpected reboot on node-{0}'.format(_id))
self.ssh_manager.execute_on_remote(ip=_ip, cmd=command)
wait(
lambda: not checkers.check_ping(self.env.get_admin_node_ip(), _ip),
timeout=60 * 10,
timeout_msg='Node {} still responds to ping'.format(
dregular_ctrl.name))
# Node don't have enough time for set offline status
# after reboot --force
# Just waiting
asserts.assert_true(
checkers.check_ping(self.env.get_admin_node_ip(),
_ip,
deadline=600),
"Host {0} is not reachable by ping during 600 sec".format(_ip))
self.fuel_web.wait_node_is_online(dregular_ctrl)
logger.info('Check that node-{0} not in maintenance mode after'
' unexpected reboot'.format(_id))
wait(lambda: tcp_ping(_ip, 22),
timeout=60 * 10,
timeout_msg='Node {} still is not available by SSH'.format(
dregular_ctrl.name))
asserts.assert_false('True' in check_auto_mode(_ip),
"Maintenance mode should not switched")
# Wait until MySQL Galera is UP on some controller
self.fuel_web.wait_mysql_galera_is_up([dregular_ctrl.name])
# Wait until Cinder services UP on a controller
self.fuel_web.wait_cinder_is_up([dregular_ctrl.name])
# Wait until RabbitMQ cluster is UP
wait_pass(lambda: self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['ha'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'RabbitMQ availability')),
timeout=1500)
logger.info('RabbitMQ cluster is available')
# TODO(astudenov): add timeout_msg
wait_pass(lambda: self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['sanity'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'Check that required services are running')),
timeout=1500)
logger.info("Required services are running")
try:
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 600 second try one more time"
" and if it fails again - test will fails ")
time.sleep(600)
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
def ha_corosync_stability_check(self):
@logwrap
def _get_pcm_nodes(remote, pure=False):
nodes = {}
pcs_status = remote.execute('pcs status nodes')['stdout']
pcm_nodes = yaml.load(''.join(pcs_status).strip())
for status in ('Online', 'Offline', 'Standby'):
list_nodes = (pcm_nodes['Pacemaker Nodes'][status]
or '').split()
if not pure:
nodes[status] = [
self.fuel_web.get_fqdn_by_hostname(x)
for x in list_nodes
]
else:
nodes[status] = list_nodes
return nodes
def _check_all_pcs_nodes_status(ctrl_remotes, pcs_nodes_online,
status):
for remote in ctrl_remotes:
pcs_nodes = _get_pcm_nodes(remote)
logger.debug(
"Status of pacemaker nodes on node {0}: {1}".format(
node['name'], pcs_nodes))
if set(pcs_nodes_online) != set(pcs_nodes[status]):
return False
return True
if not self.env.d_env.has_snapshot(self.snapshot_name):
raise SkipTest()
self.env.revert_snapshot(self.snapshot_name)
devops_name = self.env.d_env.nodes().slaves[0].name
controller_node = self.fuel_web.get_nailgun_node_by_name(devops_name)
with self.fuel_web.get_ssh_for_node(devops_name) as remote_controller:
pcs_nodes = self.fuel_web.get_pcm_nodes(devops_name)
assert_true(
not pcs_nodes['Offline'],
"There are offline nodes: {0}".format(pcs_nodes['Offline']))
pcs_nodes_online = pcs_nodes['Online']
cluster_id = self.fuel_web.get_last_created_cluster()
ctrl_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['controller'])
alive_corosync_nodes = [
node for node in ctrl_nodes
if node['mac'] != controller_node['mac']
]
ctrl_remotes = [
self.env.d_env.get_ssh_to_remote(node['ip'])
for node in ctrl_nodes
]
live_remotes = [
self.env.d_env.get_ssh_to_remote(node['ip'])
for node in alive_corosync_nodes
]
for count in xrange(500):
logger.debug('Checking splitbrain in the loop, '
'count number: {0}'.format(count))
_wait(lambda: assert_equal(
remote_controller.execute('killall -TERM corosync')
['exit_code'], 0, 'Corosync was not killed on controller, '
'see debug log, count-{0}'.format(count)),
timeout=20)
_wait(lambda: assert_true(
_check_all_pcs_nodes_status(live_remotes, [
controller_node['fqdn']
], 'Offline'), 'Caught splitbrain, see debug log, '
'count-{0}'.format(count)),
timeout=20)
_wait(lambda: assert_equal(
remote_controller.execute(
'service corosync start && service pacemaker '
'restart')['exit_code'], 0,
'Corosync was not started, see debug log,'
' count-{0}'.format(count)),
timeout=20)
_wait(lambda: assert_true(
_check_all_pcs_nodes_status(ctrl_remotes, pcs_nodes_online,
'Online'),
'Corosync was not started on controller, see debug '
'log, count: {0}'.format(count)),
timeout=20)
for remote in ctrl_remotes:
remote.clear()
for remote in live_remotes:
remote.clear()
def __check_plugin_v4_on_node(self, node="slave-01"):
logger.debug("Start to check service on node {0}".format(node))
ip = self.fuel_web.get_node_ip_by_devops_name(node)
self.ssh_manager.execute_on_remote(ip, 'pgrep -f fuel-simple-service')
self.ssh_manager.execute_on_remote(ip, 'curl localhost:8234')
def manual_cic_maintenance_mode(self):
"""Check manual maintenance mode for controller
Scenario:
1. Revert snapshot
2. Switch in maintenance mode
3. Wait until controller is rebooting
4. Exit maintenance mode
5. Check the controller become available
Duration 155m
"""
self.env.revert_snapshot('cic_maintenance_mode')
cluster_id = self.fuel_web.get_last_created_cluster()
# Select a non-primary controller
regular_ctrl = self.fuel_web.get_nailgun_node_by_name("slave-02")
dregular_ctrl = self.fuel_web.get_devops_node_by_nailgun_node(
regular_ctrl)
_ip = regular_ctrl['ip']
_id = regular_ctrl['id']
logger.info('Maintenance mode for node-{0}'.format(_id))
asserts.assert_true('True' in check_available_mode(_ip),
"Maintenance mode is not available")
self.ssh_manager.execute_on_remote(ip=_ip, cmd="umm on")
self.fuel_web.wait_node_is_offline(dregular_ctrl)
asserts.assert_true(
checkers.check_ping(self.env.get_admin_node_ip(),
_ip,
deadline=600),
"Host {0} is not reachable by ping during 600 sec".format(_ip))
asserts.assert_true('True' in check_auto_mode(_ip),
"Maintenance mode is not switched on")
self.ssh_manager.execute_on_remote(ip=_ip, cmd="umm off")
self.fuel_web.wait_node_is_online(dregular_ctrl)
# Wait until Cinder services UP on a controller
self.fuel_web.wait_cinder_is_up([dregular_ctrl.name])
# Wait until RabbitMQ cluster is UP
wait_pass(lambda: self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['ha'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'RabbitMQ availability')),
timeout=1500)
logger.info('RabbitMQ cluster is available')
wait_pass(lambda: self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['sanity'],
test_name=ostf_test_mapping.OSTF_TEST_MAPPING.get(
'Check that required services are running')),
timeout=1500)
logger.info("Required services are running")
# TODO(astudenov): add timeout_msg
try:
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 600 second try one more time"
" and if it fails again - test will fails ")
time.sleep(600)
self.fuel_web.run_ostf(cluster_id,
test_sets=['smoke', 'sanity', 'ha'])
def deploy_neutron_tun_ha_nodegroups(self):
"""Deploy HA environment with NeutronVXLAN and 2 nodegroups
Scenario:
1. Revert snapshot with ready master node
2. Bootstrap slaves from default nodegroup
3. Create cluster with Neutron VXLAN and custom nodegroups
4. Remove 2nd custom nodegroup which is added automatically
5. Bootstrap slave nodes from custom nodegroup
6. Download network configuration
7. Update network.json with customized ip ranges
8. Put new json on master node and update network data
9. Verify that new IP ranges are applied for network config
10. Add 3 controller nodes from default nodegroup
11. Add 2 compute nodes from custom nodegroup
12. Deploy cluster
13. Run network verification
14. Verify that excluded ip is not used for nodes or VIP
15. Run health checks (OSTF)
Duration 110m
Snapshot deploy_neutron_tun_ha_nodegroups
"""
if not MULTIPLE_NETWORKS:
raise SkipTest()
self.show_step(1, initialize=True)
self.env.revert_snapshot("ready")
self.show_step(2)
self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[0:3])
self.show_step(3)
cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__,
mode=DEPLOYMENT_MODE_HA,
settings={
"net_provider":
'neutron',
"net_segment_type":
NEUTRON_SEGMENT['tun'],
'tenant':
'haVxlan',
'user':
'******',
'password':
'******'
})
self.show_step(4)
self.netconf_all_groups = self.fuel_web.client.get_networks(cluster_id)
custom_group2 = self.fuel_web.get_nodegroup(cluster_id,
name=NODEGROUPS[2]['name'])
wait(lambda: not self.is_update_dnsmasq_running(self.fuel_web.client.
get_tasks()),
timeout=60,
timeout_msg="Timeout exceeded while waiting for task "
"'update_dnsmasq' is finished!")
self.fuel_web.client.delete_nodegroup(custom_group2['id'])
self.show_step(5)
self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[3:5])
self.show_step(6)
check_get_network_data_over_cli(self.ssh_manager.admin_ip, cluster_id,
'/var/log/')
management_ranges_default = []
management_ranges_custom = []
storage_ranges_default = []
storage_ranges_custom = []
default_group_id = self.fuel_web.get_nodegroup(cluster_id)['id']
custom_group_id = self.fuel_web.get_nodegroup(
cluster_id, name=NODEGROUPS[1]['name'])['id']
self.show_step(7)
with self.env.d_env.get_admin_remote() as remote:
current_net = json.loads(
remote.open('/var/log/network_1.json').read())
# Get storage ranges for default and custom groups
storage_ranges_default.append(
self.get_modified_ranges(current_net,
'storage',
group_id=default_group_id))
storage_ranges_custom.append(
self.get_modified_ranges(current_net,
'storage',
group_id=custom_group_id))
management_ranges_default.append(
self.get_modified_ranges(current_net,
'management',
group_id=default_group_id))
management_ranges_custom.append(
self.get_modified_ranges(current_net,
'management',
group_id=custom_group_id))
update_data = {
default_group_id: {
'storage': storage_ranges_default,
'management': management_ranges_default
},
custom_group_id: {
'storage': storage_ranges_custom,
'management': management_ranges_custom
}
}
updated_network = self.update_network_ranges(
current_net, update_data)
logger.debug(
'Plan to update ranges for default group to {0} for storage '
'and {1} for management and for custom group storage {2},'
' management {3}'.format(storage_ranges_default,
management_ranges_default,
storage_ranges_custom,
management_ranges_custom))
# need to push to remote
self.show_step(8)
utils.put_json_on_remote_from_dict(remote, updated_network,
cluster_id)
check_update_network_data_over_cli(self.ssh_manager.admin_ip,
cluster_id, '/var/log/')
self.show_step(9)
with self.env.d_env.get_admin_remote() as remote:
check_get_network_data_over_cli(self.ssh_manager.admin_ip,
cluster_id, '/var/log/')
latest_net = json.loads(
remote.open('/var/log/network_1.json').read())
updated_storage_default = self.get_ranges(latest_net, 'storage',
default_group_id)
updated_storage_custom = self.get_ranges(latest_net, 'storage',
custom_group_id)
updated_mgmt_default = self.get_ranges(latest_net, 'management',
default_group_id)
updated_mgmt_custom = self.get_ranges(latest_net, 'management',
custom_group_id)
asserts.assert_equal(
updated_storage_default, storage_ranges_default,
'Looks like storage range for default nodegroup '
'was not updated. Expected {0}, Actual: {1}'.format(
storage_ranges_default, updated_storage_default))
asserts.assert_equal(
updated_storage_custom, storage_ranges_custom,
'Looks like storage range for custom nodegroup '
'was not updated. Expected {0}, Actual: {1}'.format(
storage_ranges_custom, updated_storage_custom))
asserts.assert_equal(
updated_mgmt_default, management_ranges_default,
'Looks like management range for default nodegroup was '
'not updated. Expected {0}, Actual: {1}'.format(
management_ranges_default, updated_mgmt_default))
asserts.assert_equal(
updated_mgmt_custom, management_ranges_custom,
'Looks like management range for custom nodegroup was '
'not updated. Expected {0}, Actual: {1}'.format(
management_ranges_custom, updated_mgmt_custom))
self.show_step(10)
self.show_step(11)
nodegroup_default = NODEGROUPS[0]['name']
nodegroup_custom1 = NODEGROUPS[1]['name']
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': [['controller'], nodegroup_default],
'slave-02': [['controller'], nodegroup_default],
'slave-03': [['controller'], nodegroup_default],
'slave-04': [['compute', 'cinder'], nodegroup_custom1],
'slave-05': [['compute', 'cinder'], nodegroup_custom1],
})
self.show_step(12)
self.fuel_web.deploy_cluster_wait(cluster_id)
self.show_step(13)
self.fuel_web.verify_network(cluster_id)
self.show_step(14)
net_data_default_group = [
data['network_data']
for data in self.fuel_web.client.list_cluster_nodes(cluster_id)
if data['group_id'] == default_group_id
]
for net_node in net_data_default_group:
for net in net_node:
if 'storage' in net['name']:
asserts.assert_true(
self.is_ip_in_range(net['ip'].split('/')[0],
updated_storage_default[0][0],
updated_storage_default[0][-1]))
if 'management' in net['name']:
asserts.assert_true(
self.is_ip_in_range(net['ip'].split('/')[0],
updated_mgmt_default[0][0],
updated_mgmt_default[0][-1]))
net_data_custom_group = [
data['network_data']
for data in self.fuel_web.client.list_cluster_nodes(cluster_id)
if data['group_id'] == custom_group_id
]
for net_node in net_data_custom_group:
for net in net_node:
if 'storage' in net['name']:
asserts.assert_true(
self.is_ip_in_range(net['ip'].split('/')[0],
updated_storage_custom[0][0],
updated_storage_custom[0][-1]))
if 'management' in net['name']:
asserts.assert_true(
self.is_ip_in_range(net['ip'].split('/')[0],
updated_mgmt_custom[0][0],
updated_mgmt_custom[0][-1]))
mgmt_vrouter_vip = self.fuel_web.get_management_vrouter_vip(cluster_id)
logger.debug('Management vrouter vips is {0}'.format(mgmt_vrouter_vip))
mgmt_vip = self.fuel_web.get_mgmt_vip(cluster_id)
logger.debug('Management vips is {0}'.format(mgmt_vip))
# check for defaults
asserts.assert_true(
self.is_ip_in_range(
mgmt_vrouter_vip.split('/')[0], updated_mgmt_default[0][0],
updated_mgmt_default[0][-1]))
asserts.assert_true(
self.is_ip_in_range(
mgmt_vip.split('/')[0], updated_mgmt_default[0][0],
updated_mgmt_default[0][-1]))
self.show_step(15)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("deploy_neutron_tun_ha_nodegroups",
is_make=True)
def separate_rabbit_service_add_delete_node(self):
"""Add and delete rabbit node
Scenario:
1. Revert snapshot separate_rabbit_service
2. Add one rabbit node and re-deploy cluster
3. Run network verification
4. Run OSTF
5. Check hiera hosts are the same for
different group of roles
6. Delete one rabbit node
7. Run network verification
8. Run ostf
9. Check hiera hosts are the same for
different group of roles
Duration 120m
"""
self.env.revert_snapshot("separate_rabbit_service")
cluster_id = self.fuel_web.get_last_created_cluster()
node = {'slave-09': ['standalone-rabbitmq']}
self.fuel_web.update_nodes(cluster_id, node, True, False)
self.fuel_web.deploy_cluster_wait(cluster_id)
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['sanity', 'smoke', 'ha'])
checkers.check_hiera_hosts(
self,
self.fuel_web.client.list_cluster_nodes(cluster_id),
cmd='hiera amqp_hosts')
checkers.check_hiera_hosts(
self,
self.fuel_web.client.list_cluster_nodes(cluster_id),
cmd='hiera memcache_roles')
rabbit_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['standalone-rabbitmq'])
logger.debug("rabbit nodes are {0}".format(rabbit_nodes))
checkers.check_hiera_hosts(self,
rabbit_nodes,
cmd='hiera corosync_roles')
nailgun_node = self.fuel_web.update_nodes(cluster_id, node, False,
True)
nodes = filter(lambda x: x["pending_deletion"] is True, nailgun_node)
self.fuel_web.deploy_cluster_wait(cluster_id, check_services=False)
wait(lambda: self.fuel_web.is_node_discovered(nodes[0]),
timeout=6 * 60)
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['sanity', 'smoke', 'ha'])
checkers.check_hiera_hosts(
self,
self.fuel_web.client.list_cluster_nodes(cluster_id),
cmd='hiera amqp_hosts')
checkers.check_hiera_hosts(
self,
self.fuel_web.client.list_cluster_nodes(cluster_id),
cmd='hiera memcache_roles')
rabbit_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['standalone-rabbitmq'])
logger.debug("rabbit nodes are {0}".format(rabbit_nodes))
checkers.check_hiera_hosts(self,
rabbit_nodes,
cmd='hiera corosync_roles')
def deploy_simple_zabbix(self):
"""Deploy cluster in simple mode with zabbix-server
Scenario:
1. Setup master node
2. Enable 'experimental' in Nailgun
3. Restart Nailgun
4. Create cluster
5. Add 1 node with controller role
6. Add 1 node with compute role
7. Add 1 node with zabbix role
8. Deploy the cluster
9. Verify networks
10. Check that zabbix server is running on the node
11. Run OSTF
12. Login in zabbix dashboard
Snapshot: deploy_simple_zabbix
"""
self.env.revert_snapshot("ready_with_3_slaves")
node_ssh = self.env.get_ssh_to_remote(self.fuel_web.admin_node_ip)
# Turn on experimental mode
checkers.check_enable_experimental_mode(node_ssh,
'/etc/fuel/version.yaml')
# restart nailgun
checkers.restart_nailgun(node_ssh)
# check if zabbix role appears
self.fuel_web.assert_release_role_present(
release_name=hlp.OPENSTACK_RELEASE, role_name='zabbix-server')
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=hlp.DEPLOYMENT_MODE_SIMPLE,
settings={
'tenant': 'admin',
'user': '******',
'password': '******'
})
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': ['controller'],
'slave-02': ['compute'],
'slave-03': ['zabbix-server']
})
self.fuel_web.deploy_cluster_wait(cluster_id)
os_conn = os_actions.OpenStackActions(
self.fuel_web.get_nailgun_node_by_name('slave-01')['ip'])
self.fuel_web.assert_cluster_ready(os_conn,
smiles_count=6,
networks_count=1,
timeout=300)
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id)
# login in dashboard
node_ip = self.fuel_web.get_nailgun_node_by_devops_node(
self.env.get_virtual_environment().node_by_name('slave-03'))['ip']
dashboard_url = 'http://{0}/zabbix/'.format(node_ip)
logger.debug("Zabbix dashboard {0}".format(dashboard_url))
login_data = {'username': '******', 'password': '******'}
zab_client = http.HTTPClientZabbix(url=dashboard_url)
login_resp = zab_client.post(endpoint='index.php', data=login_data)
assert_equals(login_resp.code, 200)
assert_equals(login_resp.msg, 'OK')
event_resp = zab_client.get(
endpoint='events.php', cookie=login_resp.headers.get('Set-Cookie'))
assert_equals(event_resp.code, 200)
self.env.make_snapshot("deploy_simple_zabbix")
def delete_custom_nodegroup(self):
"""Delete nodegroup, check its nodes are marked as 'error'
Scenario:
1. Revert snapshot with cluster with nodes in custom nodegroup
2. Save cluster network configuration
3. Reset cluster
4. Remove custom nodegroup
5. Check nodes from custom nodegroup have 'error' status
6. Re-create custom nodegroup and upload saved network configuration
7. Assign 'error' nodes to new nodegroup
8. Check nodes from custom nodegroup are in 'discover' state
Duration 30m
"""
self.show_step(1, initialize=True)
self.env.revert_snapshot('deploy_controllers_from_custom_nodegroup')
cluster_id = self.fuel_web.get_last_created_cluster()
self.fuel_web.assert_nodes_in_ready_state(cluster_id)
self.show_step(2)
network_config = self.fuel_web.client.get_networks(cluster_id)
self.show_step(3)
custom_nodes = self.env.d_env.nodes().slaves[3:6]
self.fuel_web.stop_reset_env_wait(cluster_id)
logger.info('Waiting for all nodes online for 900 seconds...')
wait(lambda: all(n['online'] for n in self.fuel_web.client.
list_cluster_nodes(cluster_id)),
timeout=15 * 60)
self.show_step(4)
custom_nodegroup = [
ng for ng in self.fuel_web.client.get_nodegroups()
if ng['name'] == NODEGROUPS[1]['name']
][0]
self.fuel_web.client.delete_nodegroup(custom_nodegroup['id'])
self.show_step(5)
logger.info('Wait all nodes from custom nodegroup become '
'in error state..')
for slave in custom_nodes:
try:
# pylint: disable=undefined-loop-variable
wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(
slave)['status'] == 'error',
timeout=60 * 5)
# pylint: enable=undefined-loop-variable
logger.info('Node {} is in "error" state'.format(slave.name))
except TimeoutError:
raise TimeoutError('Node {} status wasn\'t changed '
'to "error"!'.format(slave.name))
self.show_step(6)
new_nodegroup = self.fuel_web.client.create_nodegroup(
cluster_id, NODEGROUPS[1]['name'])
logger.debug('Updating custom nodegroup ID in network configuration..')
network_config_new = self.fuel_web.client.get_networks(cluster_id)
for network in network_config['networks']:
if network['group_id'] == custom_nodegroup['id']:
network['group_id'] = new_nodegroup['id']
for new_network in network_config_new['networks']:
if new_network['name'] == network['name'] and \
new_network['group_id'] == network['group_id']:
network['id'] = new_network['id']
self.fuel_web.client.update_network(
cluster_id, network_config['networking_parameters'],
network_config['networks'])
self.show_step(7)
self.fuel_web.client.assign_nodegroup(new_nodegroup['id'], [
self.fuel_web.get_nailgun_node_by_devops_node(node)
for node in custom_nodes
])
self.show_step(8)
logger.info('Wait all nodes from custom nodegroup become '
'in discover state..')
for slave in custom_nodes:
try:
wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(
slave)['status'] == 'discover',
timeout=60 * 5)
logger.info('Node {} is in "discover" state'.format(
slave.name))
except TimeoutError:
raise TimeoutError('Node {} status wasn\'t changed '
'to "discover"!'.format(slave.name))
self.env.make_snapshot("delete_custom_nodegroup")
def deploy_lma_toolchain(self):
"""Deploy cluster in HA mode with the LMA toolchain
This also deploys the Elasticsearch-Kibana plugin and the
InfluxDB-Grafana plugin since they work together with the LMA collector
plugin.
Scenario:
1. Upload plugins to the master node
2. Install plugins
3. Create cluster
4. Add 3 nodes with controller role
5. Add 1 node with compute + cinder role
6. Add 1 node with influxdb_grafana + elasticsearch_kibana +
infrastructure_alerting roles
7. Deploy the cluster
8. Check that the plugins work
9. Run OSTF
Duration 150m
Snapshot deploy_lma_toolchain
"""
self.env.revert_snapshot("ready_with_5_slaves")
# TODO(scroiset): use actions fuel_actions.py
# upload_plugin and install_plugin
# copy plugins to the master node
utils.upload_tarball(ip=self.ssh_manager.admin_ip,
tar_path=settings.LMA_COLLECTOR_PLUGIN_PATH,
tar_target="/var")
utils.upload_tarball(
ip=self.ssh_manager.admin_ip,
tar_path=settings.ELASTICSEARCH_KIBANA_PLUGIN_PATH,
tar_target="/var")
utils.upload_tarball(ip=self.ssh_manager.admin_ip,
tar_path=settings.INFLUXDB_GRAFANA_PLUGIN_PATH,
tar_target="/var")
utils.upload_tarball(ip=self.ssh_manager.admin_ip,
tar_path=settings.LMA_INFRA_ALERTING_PLUGIN_PATH,
tar_target="/var")
# install plugins
utils.install_plugin_check_code(
ip=self.ssh_manager.admin_ip,
plugin=os.path.basename(settings.LMA_COLLECTOR_PLUGIN_PATH))
utils.install_plugin_check_code(
ip=self.ssh_manager.admin_ip,
plugin=os.path.basename(settings.ELASTICSEARCH_KIBANA_PLUGIN_PATH))
utils.install_plugin_check_code(
ip=self.ssh_manager.admin_ip,
plugin=os.path.basename(settings.INFLUXDB_GRAFANA_PLUGIN_PATH))
utils.install_plugin_check_code(
ip=self.ssh_manager.admin_ip,
plugin=os.path.basename(settings.LMA_INFRA_ALERTING_PLUGIN_PATH))
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
)
influxdb_user = "******"
influxdb_pass = "******"
influxdb_rootpass = "******"
grafana_user = "******"
grafana_pass = "******"
mysql_dbname = "grafanalma"
mysql_user = "******"
mysql_pass = "******"
nagios_pass = "******"
plugins = [
{
'name': 'lma_collector',
'version': '0.9.0',
'options': {
'environment_label/value': 'deploy_lma_toolchain',
'elasticsearch_mode/value': 'local',
'influxdb_mode/value': 'local',
'alerting_mode/value': 'local',
}
},
{
'name': 'elasticsearch_kibana',
'version': '0.9.0',
'options': {}
},
{
'name': 'lma_infrastructure_alerting',
'version': '0.9.0',
'options': {
'send_to/value': 'root@localhost',
'send_from/value': 'nagios@localhost',
'smtp_host/value': '127.0.0.1',
'nagios_password/value': nagios_pass,
}
},
{
'name': 'influxdb_grafana',
'version': '0.9.0',
'options': {
'influxdb_rootpass/value': influxdb_rootpass,
'influxdb_username/value': influxdb_user,
'influxdb_userpass/value': influxdb_pass,
'grafana_username/value': grafana_user,
'grafana_userpass/value': grafana_pass,
'mysql_mode/value': 'local',
'mysql_dbname/value': mysql_dbname,
'mysql_username/value': mysql_user,
'mysql_password/value': mysql_pass,
}
},
]
for plugin in plugins:
plugin_name = plugin['name']
plugin_version = plugin['version']
msg = "Plugin '{:s}' couldn't be found. " \
"Test aborted".format(plugin_name)
assert_true(
self.fuel_web.check_plugin_exists(cluster_id, plugin_name),
msg)
logger.debug('{:s} plugin is installed'.format(plugin_name))
self.fuel_web.update_plugin_settings(cluster_id, plugin_name,
plugin_version,
plugin['options'])
analytics_roles = [
"influxdb_grafana", "elasticsearch_kibana",
"infrastructure_alerting"
]
self.fuel_web.update_nodes(
cluster_id, {
"slave-01": ["controller"],
"slave-02": ["controller"],
"slave-03": ["controller"],
"slave-04": ["compute", "cinder"],
"slave-05": analytics_roles,
})
self.fuel_web.deploy_cluster_wait(cluster_id, timeout=9000)
analytics_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, analytics_roles)
msg = "One node with '{}' roles must be present, found {}".format(
' + '.join(analytics_roles), len(analytics_nodes))
assert_true(len(analytics_nodes) == 1, msg)
elasticsearch_kibana_vip = self.get_vip(cluster_id, 'es_vip_mgmt')
influxdb_grafana_vip = self.get_vip(cluster_id, 'influxdb')
nagios_vip = self.get_vip(cluster_id, 'infrastructure_alerting')
assert_is_not_none(
elasticsearch_kibana_vip,
"Fail to retrieve the Elasticsearch/Kibana cluster VIP address")
assert_is_not_none(
influxdb_grafana_vip,
"Fail to retrieve the InfluxDB/Grafana cluster VIP address")
assert_is_not_none(
nagios_vip,
"Fail to retrieve the Infrastructure Alerting cluster VIP address")
def assert_http_get_response(url, expected=200):
r = requests.get(url)
assert_equal(
r.status_code, expected,
"{} responded with {}, expected {}".format(
url, r.status_code, expected))
logger.debug("Check that Elasticsearch is ready")
assert_http_get_response(
"http://{0}:9200/".format(elasticsearch_kibana_vip))
logger.debug("Check that Kibana is ready")
assert_http_get_response(
"http://{0}/".format(elasticsearch_kibana_vip))
logger.debug("Check that the root user can access InfluxDB")
influxdb_url = "http://{0}:8086/query?db=lma&u={1}&p={2}&" + \
"q=show+measurements"
assert_http_get_response(
influxdb_url.format(influxdb_grafana_vip, 'root',
influxdb_rootpass))
logger.debug("Check that the LMA user can access InfluxDB")
assert_http_get_response(
influxdb_url.format(influxdb_grafana_vip, influxdb_user,
influxdb_pass))
logger.debug("Check that the LMA user can access Grafana")
assert_http_get_response("http://{0}:{1}@{2}:8000/api/org".format(
grafana_user, grafana_pass, influxdb_grafana_vip))
nagios_url = "http://{}:{}".format(nagios_vip, '8001')
r = requests.get(nagios_url, auth=('nagiosadmin', nagios_pass))
assert_equal(
r.status_code, 200,
"Nagios HTTP response code {}, expected {}".format(
r.status_code, 200))
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("deploy_lma_toolchain")
def add_custom_nodegroup(self):
"""Add new nodegroup to operational environment
Scenario:
1. Revert snapshot with operational cluster
2. Create new nodegroup for the environment and configure
it's networks
3. Bootstrap slave node from custom-2 nodegroup
4. Add node from new nodegroup to the environment with compute role
5. Run network verification
6. Deploy changes
7. Run network verification
8. Run OSTF
9. Check that nodes from 'default' nodegroup can reach nodes
from new nodegroup via management and storage networks
Duration 50m
Snapshot add_custom_nodegroup
"""
self.show_step(1, initialize=True)
self.env.revert_snapshot('deploy_neutron_tun_ha_nodegroups')
cluster_id = self.fuel_web.get_last_created_cluster()
self.fuel_web.assert_nodes_in_ready_state(cluster_id)
asserts.assert_true(
not any(ng['name'] == NODEGROUPS[2]['name']
for ng in self.fuel_web.client.get_nodegroups()),
'Custom nodegroup {0} already '
'exists!'.format(NODEGROUPS[2]['name']))
self.show_step(2)
new_nodegroup = self.fuel_web.client.create_nodegroup(
cluster_id, NODEGROUPS[2]['name'])
logger.debug('Updating custom nodegroup ID in network configuration..')
network_config_new = self.fuel_web.client.get_networks(cluster_id)
asserts.assert_true(self.netconf_all_groups is not None,
'Network configuration for nodegroups is empty!')
for network in self.netconf_all_groups['networks']:
if network['group_id'] is not None and \
not any(network['group_id'] == ng['id']
for ng in self.fuel_web.client.get_nodegroups()):
network['group_id'] = new_nodegroup['id']
for new_network in network_config_new['networks']:
if new_network['name'] == network['name'] and \
new_network['group_id'] == network['group_id']:
network['id'] = new_network['id']
self.fuel_web.client.update_network(
cluster_id, self.netconf_all_groups['networking_parameters'],
self.netconf_all_groups['networks'])
self.show_step(3)
self.env.bootstrap_nodes([self.env.d_env.nodes().slaves[6]])
self.show_step(4)
self.fuel_web.update_nodes(
cluster_id, {'slave-07': [['compute'], new_nodegroup['name']]},
True, False)
self.show_step(5)
self.fuel_web.verify_network(cluster_id)
self.show_step(6)
self.fuel_web.deploy_cluster_wait(cluster_id)
self.show_step(7)
self.fuel_web.verify_network(cluster_id)
self.show_step(8)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.show_step(9)
primary_ctrl = self.fuel_web.get_nailgun_node_by_devops_node(
self.fuel_web.get_nailgun_primary_node(
slave=self.env.d_env.nodes().slaves[0]))
with self.fuel_web.get_ssh_for_node('slave-07') as remote:
new_node_networks = utils.get_net_settings(remote)
for interface in ('br-storage', 'br-mgmt'):
if interface in new_node_networks:
logger.info("Checking new node is accessible from primary "
"controller via {0} interface.".format(interface))
for ip in new_node_networks[interface]['ip_addresses']:
address = ip.split('/')[0]
result = check_ping(primary_ctrl['ip'], address, timeout=3)
asserts.assert_true(
result, "New node isn't accessible from "
"primary controller via {0} interface"
": {1}.".format(interface, result))
self.env.make_snapshot("add_custom_nodegroup")
def deploy_ceilometer_ha_one_controller_with_mongo(self):
"""Deploy cluster in HA mode with Ceilometer
Scenario:
1. Create cluster. Set install Ceilometer option
2. Add 1 node with controller role
3. Add 1 nodes with compute role
4. Add 1 node with cinder role
5. Add 1 node with mongo role
6. Deploy the cluster
7. Verify ceilometer api is running
8. Run OSTF
Duration 45m
Snapshot: deploy_ceilometer_ha_one_controller_with_mongo
"""
self.env.revert_snapshot("ready_with_3_slaves")
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings={
'ceilometer': True,
'net_provider': 'neutron',
'net_segment_type': 'tun',
'tenant': 'ceilometerSimple',
'user': '******',
'password': '******'
})
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': ['controller'],
'slave-02': ['compute', 'cinder'],
'slave-03': ['mongo']
})
nailgun_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
disk_mb = 0
for node in nailgun_nodes:
if node.get('pending_roles') == ['mongo']:
disk_mb = self.fuel_web.get_node_disk_size(
node.get('id'), "vda")
logger.debug('disk size is {0}'.format(disk_mb))
mongo_disk_mb = 11116
os_disk_mb = disk_mb - mongo_disk_mb
mongo_disk_gb = ("{0}G".format(round(mongo_disk_mb / 1024, 1)))
disk_part = {"vda": {"os": os_disk_mb, "mongo": mongo_disk_mb}}
for node in nailgun_nodes:
if node.get('pending_roles') == ['mongo']:
self.fuel_web.update_node_disk(node.get('id'), disk_part)
self.fuel_web.deploy_cluster_wait(cluster_id)
_ip = self.fuel_web.get_nailgun_node_by_name("slave-01")['ip']
with self.env.d_env.get_ssh_to_remote(_ip) as remote:
checkers.verify_service(remote,
service_name='ceilometer-api',
ignore_count_of_proccesses=True)
_ip = self.fuel_web.get_nailgun_node_by_name("slave-03")['ip']
with self.env.d_env.get_ssh_to_remote(_ip) as remote:
partitions = checkers.get_mongo_partitions(remote, "vda5")
assert_equal(
partitions[0].rstrip(), mongo_disk_gb,
'Mongo size {0} before deployment is not equal'
' to size after {1}'.format(mongo_disk_gb, partitions))
self.run_tests(cluster_id)
self.env.make_snapshot(
"deploy_ceilometer_ha_one_controller_with_mongo")
def execute_through_host(self, ssh, vm_host, cmd, creds=()):
logger.debug("Making intermediate transport")
intermediate_transport = ssh._ssh.get_transport()
logger.debug("Opening channel to VM")
intermediate_channel = intermediate_transport.open_channel(
'direct-tcpip', (vm_host, 22), (ssh.host, 0))
logger.debug("Opening paramiko transport")
transport = paramiko.Transport(intermediate_channel)
logger.debug("Starting client")
transport.start_client()
logger.info("Passing authentication to VM: {}".format(creds))
if not creds:
creds = ('cirros', 'cubswin:)')
transport.auth_password(creds[0], creds[1])
logger.debug("Opening session")
channel = transport.open_session()
logger.info("Executing command: {}".format(cmd))
channel.exec_command(cmd)
result = {
'stdout': [],
'stderr': [],
'exit_code': 0
}
logger.debug("Receiving exit_code")
result['exit_code'] = channel.recv_exit_status()
logger.debug("Receiving stdout")
result['stdout'] = channel.recv(1024)
logger.debug("Receiving stderr")
result['stderr'] = channel.recv_stderr(1024)
logger.debug("Closing channel")
channel.close()
return result
def deploy_murano_ha_with_tun(self):
"""Deploy cluster in ha mode with Murano and Neutron VXLAN
Scenario:
1. Create cluster. Set install Murano option
2. Add 3 node with controller role
3. Add 1 nodes with compute role
4. Deploy the cluster
5. Verify Murano services
6. Run OSTF
7. Run OSTF Murano platform tests
Duration 100m
Snapshot: deploy_murano_ha_with_tun
"""
self.env.revert_snapshot("ready_with_5_slaves")
data = {
'murano': True,
'net_provider': 'neutron',
'net_segment_type': settings.NEUTRON_SEGMENT['tun'],
'tenant': 'muranoHA',
'user': '******',
'password': '******'
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings=data)
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute']
})
self.fuel_web.deploy_cluster_wait(cluster_id)
cluster_vip = self.fuel_web.get_public_vip(cluster_id)
os_conn = os_actions.OpenStackActions(cluster_vip, data['user'],
data['password'], data['tenant'])
self.fuel_web.assert_cluster_ready(os_conn, smiles_count=13)
for slave in ["slave-01", "slave-02", "slave-03"]:
_ip = self.fuel_web.get_nailgun_node_by_name(slave)['ip']
with self.env.d_env.get_ssh_to_remote(_ip) as remote:
checkers.verify_service(remote, service_name='murano-api')
logger.debug('Run sanity and functional Murano OSTF tests')
self.fuel_web.run_single_ostf_test(
cluster_id=self.fuel_web.get_last_created_cluster(),
test_sets=['sanity'],
test_name=('fuel_health.tests.sanity.test_sanity_murano.'
'MuranoSanityTests.test_create_and_delete_service'))
logger.debug('Run OSTF platform tests')
test_class_main = ('fuel_health.tests.tests_platform'
'.test_murano_linux.MuranoDeployLinuxServicesTests')
tests_names = [
'test_deploy_dummy_app',
]
test_classes = []
for test_name in tests_names:
test_classes.append('{0}.{1}'.format(test_class_main, test_name))
for test_name in test_classes:
self.fuel_web.run_single_ostf_test(cluster_id=cluster_id,
test_sets=['tests_platform'],
test_name=test_name,
timeout=60 * 20)
self.env.make_snapshot("deploy_murano_ha_with_tun")
def deploy_heat_ha(self):
"""Deploy Heat cluster in HA mode
Scenario:
1. Create cluster
2. Add 3 node with controller role and mongo
3. Add 1 nodes with compute role
4. Set Ceilometer install option
5. Deploy the cluster
6. Verify Heat and Ceilometer services
7. Run OSTF platform tests
Duration 70m
Snapshot: deploy_heat_ha
"""
self.env.revert_snapshot("ready_with_5_slaves")
data = {
'ceilometer': True,
'net_provider': 'neutron',
'net_segment_type': settings.NEUTRON_SEGMENT['tun'],
'tenant': 'heatHA',
'user': '******',
'password': '******'
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings=data)
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': ['controller', 'mongo'],
'slave-02': ['controller', 'mongo'],
'slave-03': ['controller', 'mongo'],
'slave-04': ['compute']
})
self.fuel_web.deploy_cluster_wait(cluster_id)
cluster_vip = self.fuel_web.get_public_vip(cluster_id)
os_conn = os_actions.OpenStackActions(cluster_vip, data['user'],
data['password'], data['tenant'])
self.fuel_web.assert_cluster_ready(os_conn, smiles_count=13)
for slave in ["slave-01", "slave-02", "slave-03"]:
_ip = self.fuel_web.get_nailgun_node_by_name(slave)['ip']
with self.env.d_env.get_ssh_to_remote(_ip) as remote:
checkers.verify_service(remote,
service_name='heat-api',
count=3)
checkers.verify_service(remote,
service_name='ceilometer-api',
ignore_count_of_proccesses=True)
logger.debug('Run Heat OSTF platform tests')
test_class_main = ('fuel_health.tests.tests_platform.'
'test_heat.'
'HeatSmokeTests')
tests_names = [
'test_actions', 'test_advanced_actions', 'test_autoscaling',
'test_rollback', 'test_update'
]
test_classes = []
for test_name in tests_names:
test_classes.append('{0}.{1}'.format(test_class_main, test_name))
for test_name in test_classes:
self.fuel_web.run_single_ostf_test(cluster_id=cluster_id,
test_sets=['tests_platform'],
test_name=test_name,
timeout=60 * 60)
self.env.make_snapshot("deploy_heat_ha")
def patching_environment(self):
"""Apply patches on deployed environment
Scenario:
1. Revert snapshot of deployed environment
2. Run Rally benchmark tests and store results
3. Download patched packages on master node and make local repositories
4. Add new local repositories on slave nodes
5. Download late artifacts and clean generated images if needed
6. Perform actions required to apply patches
7. Verify that fix works
8. Run OSTF
9. Run Rally benchmark tests and compare results
Duration 15m
"""
# Step #1
if not self.env.revert_snapshot(self.snapshot_name):
raise PatchingTestException('Environment revert from snapshot "{0}'
'" failed.'.format(self.snapshot_name))
# Check that environment exists and it's ready for patching
cluster_id = self.fuel_web.get_last_created_cluster()
assert_is_not_none(cluster_id, 'Environment for patching not found.')
# Step #2
if settings.PATCHING_RUN_RALLY:
rally_benchmarks = {}
benchmark_results1 = {}
for tag in set(settings.RALLY_TAGS):
rally_benchmarks[tag] = RallyBenchmarkTest(
container_repo=settings.RALLY_DOCKER_REPO,
environment=self.env,
cluster_id=cluster_id,
test_type=tag
)
benchmark_results1[tag] = rally_benchmarks[tag].run()
logger.debug(benchmark_results1[tag].show())
# Step #3
patching_repos = patching.add_remote_repositories(
self.env, settings.PATCHING_MIRRORS)
if settings.PATCHING_MASTER_MIRRORS:
patching_master_repos = patching.add_remote_repositories(
self.env, settings.PATCHING_MASTER_MIRRORS,
prefix_name='custom_master_repo')
# Step #4
slaves = self.fuel_web.client.list_cluster_nodes(cluster_id)
for repo in patching_repos:
patching.connect_slaves_to_repo(self.env, slaves, repo)
if settings.PATCHING_MASTER_MIRRORS:
with self.env.d_env.get_admin_remote() as remote:
for repo in patching_master_repos:
install_pkg(remote, 'yum-utils')
patching.connect_admin_to_repo(self.env, repo)
# Step #5
if settings.LATE_ARTIFACTS_JOB_URL:
data = urlopen(settings.LATE_ARTIFACTS_JOB_URL +
"/artifact/artifacts/artifacts.txt")
for package in data:
os.system("wget --directory-prefix"
" {0} {1}".format(settings.UPDATE_FUEL_PATH,
package))
self.env.admin_actions.upload_packages(
local_packages_dir=settings.UPDATE_FUEL_PATH,
centos_repo_path='/var/www/nailgun/centos/auxiliary',
ubuntu_repo_path=settings.LOCAL_MIRROR_UBUNTU)
if settings.REGENERATE_ENV_IMAGE:
self.env.admin_actions.clean_generated_image(
settings.OPENSTACK_RELEASE)
# Step #6
logger.info('Applying fix...')
patching.apply_patches(self.env, target='environment', slaves=slaves)
# Step #7
logger.info('Verifying fix...')
patching.verify_fix(self.env, target='environment', slaves=slaves)
# Step #8
# If OSTF fails (sometimes services aren't ready after
# slaves nodes reboot) sleep 5 minutes and try again
try:
self.fuel_web.run_ostf(cluster_id=cluster_id)
except AssertionError:
time.sleep(300)
self.fuel_web.run_ostf(cluster_id=cluster_id)
# Step #9
if settings.PATCHING_RUN_RALLY:
benchmark_results2 = {}
for tag in set(settings.RALLY_TAGS):
benchmark_results2[tag] = rally_benchmarks[tag].run()
logger.debug(benchmark_results2[tag].show())
rally_benchmarks_passed = True
for tag in set(settings.RALLY_TAGS):
if not RallyResult.compare(benchmark_results1[tag],
benchmark_results2[tag],
deviation=0.2):
rally_benchmarks_passed = False
assert_true(rally_benchmarks_passed,
"Rally benchmarks show performance degradation "
"after packages patching.")
number_of_nodes = len(self.fuel_web.client.list_cluster_nodes(
cluster_id))
cluster_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
roles_list = [node['roles'] for node in cluster_nodes]
unique_roles = []
for role in roles_list:
if not [unique_role for unique_role in unique_roles
if set(role) == set(unique_role)]:
unique_roles.append(role)
self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[
number_of_nodes:number_of_nodes + 1])
for roles in unique_roles:
if "mongo" in roles:
continue
node = {'slave-0{}'.format(number_of_nodes + 1):
[role for role in roles]}
logger.debug("Adding new node to the cluster: {0}".format(node))
self.fuel_web.update_nodes(
cluster_id, node)
self.fuel_web.deploy_cluster_wait(cluster_id,
check_services=False)
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['sanity', 'smoke', 'ha'])
if "ceph-osd" in roles:
with self.fuel_web.get_ssh_for_node(
'slave-0{}'.format(number_of_nodes + 1)
) as remote_ceph:
self.fuel_web.prepare_ceph_to_delete(remote_ceph)
nailgun_node = self.fuel_web.update_nodes(
cluster_id, node, False, True)
nodes = [_node for _node in nailgun_node
if _node["pending_deletion"] is True]
self.fuel_web.deploy_cluster(cluster_id)
self.fuel_web.wait_node_is_discovered(nodes[0])
# sanity set isn't running due to LP1457515
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['smoke', 'ha'])
def deploy_sahara_ha_one_controller_tun(self):
"""Deploy cluster in ha mode with 1 controller Sahara and Neutron VXLAN
Scenario:
1. Create a Fuel cluster. Set the option for Sahara installation
2. Add 1 node with "controller" role
3. Add 1 node with "compute" role
4. Deploy the Fuel cluster
5. Verify Sahara service on controller
6. Run all sanity and smoke tests
7. Register Vanilla2 image for Sahara
8. Run platform Vanilla2 test for Sahara
Duration 65m
Snapshot: deploy_sahara_ha_one_controller_tun
"""
self.env.revert_snapshot("ready_with_3_slaves")
logger.debug('Create Fuel cluster for Sahara tests')
data = {
'sahara': True,
'net_provider': 'neutron',
'net_segment_type': settings.NEUTRON_SEGMENT['tun'],
'tenant': 'saharaSimple',
'user': '******',
'password': '******'
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings=data)
self.fuel_web.update_nodes(cluster_id, {
'slave-01': ['controller'],
'slave-02': ['compute']
})
self.fuel_web.deploy_cluster_wait(cluster_id)
os_conn = os_actions.OpenStackActions(
self.fuel_web.get_public_vip(cluster_id), data['user'],
data['password'], data['tenant'])
self.fuel_web.assert_cluster_ready(os_conn, smiles_count=5)
logger.debug('Verify Sahara service on controller')
_ip = self.fuel_web.get_nailgun_node_by_name("slave-01")['ip']
with self.env.d_env.get_ssh_to_remote(_ip) as remote:
checkers.verify_service(remote, service_name='sahara-api')
checkers.verify_service(remote, service_name='sahara-engine')
logger.debug('Check MD5 sum of Vanilla2 image')
check_image = checkers.check_image(
settings.SERVTEST_SAHARA_VANILLA_2_IMAGE,
settings.SERVTEST_SAHARA_VANILLA_2_IMAGE_MD5,
settings.SERVTEST_LOCAL_PATH)
asserts.assert_true(check_image)
logger.debug('Run all sanity and smoke tests')
path_to_tests = 'fuel_health.tests.sanity.test_sanity_sahara.'
test_names = [
'VanillaTwoTemplatesTest.test_vanilla_two_templates',
'HDPTwoTemplatesTest.test_hdp_two_templates'
]
self.fuel_web.run_ostf(
cluster_id=self.fuel_web.get_last_created_cluster(),
tests_must_be_passed=[
path_to_tests + test_name for test_name in test_names
])
logger.debug('Import Vanilla2 image for Sahara')
with open('{0}/{1}'.format(
settings.SERVTEST_LOCAL_PATH,
settings.SERVTEST_SAHARA_VANILLA_2_IMAGE)) as data:
os_conn.create_image(
name=settings.SERVTEST_SAHARA_VANILLA_2_IMAGE_NAME,
properties=settings.SERVTEST_SAHARA_VANILLA_2_IMAGE_META,
data=data,
is_public=True,
disk_format='qcow2',
container_format='bare')
path_to_tests = 'fuel_health.tests.tests_platform.test_sahara.'
test_names = ['VanillaTwoClusterTest.test_vanilla_two_cluster']
for test_name in test_names:
logger.debug('Run platform test {0} for Sahara'.format(test_name))
self.fuel_web.run_single_ostf_test(cluster_id=cluster_id,
test_sets=['tests_platform'],
test_name=path_to_tests +
test_name,
timeout=60 * 200)
self.env.make_snapshot("deploy_sahara_ha_one_controller_tun")
def get_slave_kernel(cls, slave_remote):
kernel = ''.join(slave_remote.execute(
r"uname -r | sed -rn"
r" 's/^([0-9, \.]+(\-[0-9]+)?)-.*/\1/p'")['stdout']).rstrip()
logger.debug("slave kernel is {0}".format(kernel))
return kernel
def ha_delete_vips(self):
"""Delete all management and public VIPs on all controller nodes.
Verify that they are restored.
Verify total amount of secondary IPs. Should be 2:
management and public
Scenario:
1. Delete all secondary VIP
2. Wait while it is being restored
3. Verify it is restored
Snapshot deploy_ha
"""
logger.debug('Start reverting of deploy_ha snapshot')
self.env.revert_snapshot("deploy_ha")
cluster_id = \
self.fuel_web.client.get_cluster_id(self.__class__.__name__)
logger.debug('Cluster id is {0}'.format(cluster_id))
interfaces = ('hapr-p', 'hapr-m')
slaves = self.env.nodes().slaves[:3]
logger.debug("Current nodes are {0}".format([i.name for i in slaves]))
ips_amount = 0
for devops_node in slaves:
# Verify VIPs are started.
ret = self.fuel_web.get_pacemaker_status(devops_node.name)
logger.debug("Pacemaker status {0} for node {1}".format
(ret, devops_node.name))
assert_true(
re.search('vip__management_old\s+\(ocf::mirantis:ns_IPaddr2\):'
'\s+Started node', ret),
'vip management not started. '
'Current pacemaker status is {0}'.format(ret))
assert_true(
re.search('vip__public_old\s+\(ocf::mirantis:ns_IPaddr2\):'
'\s+Started node', ret),
'vip public not started. '
'Current pacemaker status is {0}'.format(ret))
for interface in interfaces:
# Look for management and public ip in namespace and remove it
logger.debug("Start to looking for ip of Vips")
addresses = self.fuel_web.ip_address_show(
devops_node.name, interface=interface,
namespace='haproxy',
pipe_str='| grep {0}$'.format(interface))
logger.debug("Vip addresses is {0} for node {1} and interface"
" {2}".format(addresses, devops_node.name,
interface))
ip_search = re.search(
'inet (?P<ip>\d+\.\d+\.\d+.\d+/\d+) scope global '
'{0}'.format(interface), addresses)
if ip_search is None:
logger.debug("Ip show output does not"
" match in regex. Current value is None")
continue
ip = ip_search.group('ip')
logger.debug("Founded ip is {0}".format(ip))
logger.debug("Start ip {0} deletion on node {1} and "
"interface {2} ".format(ip, devops_node.name,
interface))
self.fuel_web.ip_address_del(
node_name=devops_node.name,
interface=interface,
ip=ip, namespace='haproxy')
# The ip should be restored
ip_assigned = lambda nodes: \
any([ip in self.fuel_web.ip_address_show(
n.name, 'haproxy',
interface, '| grep {0}$'.format(interface))
for n in nodes])
logger.debug("Waiting while deleted ip restores ...")
wait(lambda: ip_assigned(slaves), timeout=30)
assert_true(ip_assigned(slaves),
"IP isn't restored restored.")
ips_amount += 1
time.sleep(60)
# Run OSTF tests
failed_test_name = ['Create volume and boot instance from it',
'Create volume and attach it to instance']
self.fuel_web.run_ostf(
cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'],
should_fail=3,
failed_test_name=failed_test_name)
# Revert initial state. VIP could be moved to other controller
self.env.revert_snapshot("deploy_ha")
assert_equal(ips_amount, 2,
'Not all vips were recovered after fail in 10s')
def deploy_murano_ha_with_tun(self):
"""Deploy cluster in ha mode with Murano and Neutron VXLAN
Scenario:
1. Create cluster. Set install Murano option
2. Add 3 node with controller role
3. Add 1 nodes with compute role
4. Deploy the cluster
5. Verify Murano services
6. Run OSTF
7. Run OSTF Murano platform tests
Duration 100m
Snapshot: deploy_murano_ha_with_tun
"""
self.env.revert_snapshot("ready_with_5_slaves")
data = {
'murano': True,
'net_provider': 'neutron',
'net_segment_type': settings.NEUTRON_SEGMENT['tun'],
'tenant': 'muranoHA',
'user': '******',
'password': '******'
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings=data,
configure_ssl=False)
# TODO(freerunner): Need to configure SSL again when root cause for
# TODO(freerunner): https://bugs.launchpad.net/fuel/+bug/1590633
# TODO(freerunner): will be found and fixed
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute']
})
self.fuel_web.deploy_cluster_wait(cluster_id)
for slave in ["slave-01", "slave-02", "slave-03"]:
_ip = self.fuel_web.get_nailgun_node_by_name(slave)['ip']
checkers.verify_service(_ip, service_name='murano-api')
logger.debug('Run sanity and functional Murano OSTF tests')
self.fuel_web.run_ostf(cluster_id=cluster_id, test_sets=['sanity'])
logger.debug('Run OSTF platform tests')
test_class_main = ('fuel_health.tests.tests_platform'
'.test_murano_linux.MuranoDeployLinuxServicesTests')
tests_names = ['test_deploy_dummy_app_with_glare']
test_classes = []
for test_name in tests_names:
test_classes.append('{0}.{1}'.format(test_class_main, test_name))
for test_name in test_classes:
self.fuel_web.run_single_ostf_test(cluster_id=cluster_id,
test_sets=['tests_platform'],
test_name=test_name,
timeout=60 * 20)
self.env.make_snapshot("deploy_murano_ha_with_tun")
def patching_test(self):
"""Apply patches on deployed master
Scenario:
1. Download patched packages on master node and make local repositories
2. Download late artifacts and clean generated images if needed
3. Perform actions required to apply patches
4. Verify that fix works
5. Run OSTF
6. Run network verification
7. Reset and delete cluster
8. Bootstrap 3 slaves
Duration 30m
"""
if not self.env.revert_snapshot(self.snapshot_name):
raise PatchingTestException('Environment revert from snapshot "{0}'
'" failed.'.format(self.snapshot_name))
# Step #1
with self.env.d_env.get_admin_remote() as remote:
install_pkg(remote, 'yum-utils')
patching_repos = patching.add_remote_repositories(
self.env, settings.PATCHING_MASTER_MIRRORS)
for repo in patching_repos:
patching.connect_admin_to_repo(self.env, repo)
# Step #2
if settings.LATE_ARTIFACTS_JOB_URL:
data = urlopen(
settings.LATE_ARTIFACTS_JOB_URL +
"/artifact/artifacts/artifacts.txt")
for package in data:
os.system("wget --directory-prefix"
" {0} {1}".format(settings.UPDATE_FUEL_PATH,
package))
self.env.admin_actions.upload_packages(
local_packages_dir=settings.UPDATE_FUEL_PATH,
centos_repo_path='/var/www/nailgun/centos/auxiliary',
ubuntu_repo_path=settings.LOCAL_MIRROR_UBUNTU)
if settings.REGENERATE_ENV_IMAGE:
self.env.admin_actions.clean_generated_image(
settings.OPENSTACK_RELEASE)
# Step #3
logger.info('Applying fix...')
patching.apply_patches(self.env, target='master')
# Step #4
logger.info('Verifying fix...')
patching.verify_fix(self.env, target='master')
# Step #5
active_nodes = []
for node in self.env.d_env.nodes().slaves:
if node.driver.node_active(node):
active_nodes.append(node)
logger.debug('active nodes are {}'.format(active_nodes))
cluster_id = self.fuel_web.get_last_created_cluster()
if self.fuel_web.get_last_created_cluster():
number_of_nodes = len(self.fuel_web.client.list_cluster_nodes(
cluster_id))
self.fuel_web.run_ostf(cluster_id=cluster_id)
if number_of_nodes > 1:
self.fuel_web.verify_network(cluster_id)
cluster_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
roles_list = [node['roles'] for node in cluster_nodes]
unique_roles = []
for role in roles_list:
if not [unique_role for unique_role in unique_roles
if set(role) == set(unique_role)]:
unique_roles.append(role)
self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[
number_of_nodes:number_of_nodes + 1])
for roles in unique_roles:
if "mongo" in roles:
continue
node = {'slave-0{}'.format(number_of_nodes + 1):
[role for role in roles]}
logger.debug("Adding new node to"
" the cluster: {0}".format(node))
self.fuel_web.update_nodes(
cluster_id, node)
self.fuel_web.deploy_cluster_wait(cluster_id,
check_services=False)
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['sanity', 'smoke', 'ha'])
if "ceph-osd" in roles:
with self.fuel_web.get_ssh_for_node(
'slave-0{}'.format(number_of_nodes + 1)
) as remote:
self.fuel_web.prepare_ceph_to_delete(remote)
nailgun_node = self.fuel_web.update_nodes(
cluster_id, node, False, True)
nodes = [_node for _node in nailgun_node
if _node["pending_deletion"] is True]
self.fuel_web.deploy_cluster(cluster_id)
self.fuel_web.wait_node_is_discovered(nodes[0])
# sanity set isn't running due to LP1457515
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['smoke', 'ha'])
active_nodes = []
for node in self.env.d_env.nodes().slaves:
if node.driver.node_active(node):
active_nodes.append(node)
logger.debug('active nodes are {}'.format(active_nodes))
self.fuel_web.stop_reset_env_wait(cluster_id)
self.fuel_web.wait_nodes_get_online_state(
active_nodes, timeout=10 * 60)
self.fuel_web.client.delete_cluster(cluster_id)
wait((lambda: len(
self.fuel_web.client.list_nodes()) == number_of_nodes),
timeout=5 * 60,
timeout_msg='Timeout: Nodes are not discovered')
self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[:3])
def list_tasks(self):
cmd = "rally task list --uuids-only"
result = self.run_container_command(cmd)
logger.debug('Rally tasks list: {0}'.format(result))
return [line.strip() for line in result['stdout']]
def rollback_automatically_simple_env(self):
"""Rollback automatically simple deployed cluster
Scenario:
1. Revert snapshot with simple neutron gre env
2. Add raise exception to docker_engine.py file
3. Run upgrade on master
4. Check that rollback starts automatically
5. Check that cluster was not upgraded and run OSTf
6. Add 1 cinder node and re-deploy cluster
7. Run OSTF
"""
if not self.env.get_virtual_environment().has_snapshot(
'deploy_neutron_gre'):
raise SkipTest()
self.env.revert_snapshot("deploy_neutron_gre")
cluster_id = self.fuel_web.get_last_created_cluster()
remote = self.env.get_ssh_to_remote_by_name('slave-01')
expected_kernel = UpgradeFuelMaster.get_slave_kernel(remote)
checkers.upload_tarball(self.env.get_admin_remote(),
hlp_data.TARBALL_PATH, '/var')
checkers.check_tarball_exists(self.env.get_admin_remote(),
os.path.basename(hlp_data.
TARBALL_PATH),
'/var')
checkers.untar(self.env.get_admin_remote(),
os.path.basename(hlp_data.
TARBALL_PATH), '/var')
self.fuel_web.modify_python_file(self.env.get_admin_remote(),
"2i \ \ \ \ 2014.2-6.0: blah-blah",
'/var/upgrade/releases/'
'metadata.yaml')
#we expect 255 exit code here because upgrade failed
# and exit status is 255
checkers.run_script(self.env.get_admin_remote(), '/var', 'upgrade.sh',
password=
hlp_data.KEYSTONE_CREDS['password'],
rollback=True, exit_code=255)
checkers.wait_rollback_is_done(self.env.get_admin_remote(), 3000)
checkers.check_upgraded_containers(self.env.get_admin_remote(),
hlp_data.UPGRADE_FUEL_TO,
hlp_data.UPGRADE_FUEL_FROM)
logger.debug("all containers are ok")
_wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(
self.env.nodes().slaves[0]), timeout=120)
logger.debug("all services are up now")
self.fuel_web.wait_nodes_get_online_state(self.env.nodes().slaves[:3])
self.fuel_web.assert_nodes_in_ready_state(cluster_id)
self.fuel_web.assert_fuel_version(hlp_data.UPGRADE_FUEL_FROM)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.bootstrap_nodes(self.env.nodes().slaves[3:4])
self.fuel_web.update_nodes(
cluster_id, {'slave-04': ['cinder']},
True, False
)
self.fuel_web.deploy_cluster_wait(cluster_id)
if hlp_data.OPENSTACK_RELEASE_UBUNTU in hlp_data.OPENSTACK_RELEASE:
remote = self.env.get_ssh_to_remote_by_name('slave-04')
kernel = UpgradeFuelMaster.get_slave_kernel(remote)
checkers.check_kernel(kernel, expected_kernel)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("rollback_automatic_simple")
