def ceph_multinode_with_cinder(self):
"""Deploy ceph with cinder in simple mode
Scenario:
1. Create cluster
2. Add 1 node with controller role
3. Add 1 node with compute role
4. Add 2 nodes with cinder and ceph OSD roles
5. Deploy the cluster
6. Check ceph status
7. Check partitions on controller node
Snapshot ceph_multinode_with_cinder
"""
if settings.OPENSTACK_RELEASE == settings.OPENSTACK_RELEASE_REDHAT:
raise SkipTest()
self.env.revert_snapshot("ready")
self.env.bootstrap_nodes(self.env.nodes().slaves[:4])
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE_SIMPLE,
settings={
'volumes_ceph': False,
'images_ceph': True,
'volumes_lvm': True
}
)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller'],
'slave-02': ['compute'],
'slave-03': ['cinder', 'ceph-osd'],
'slave-04': ['cinder', 'ceph-osd']
}
)
# Cluster deploy
self.fuel_web.deploy_cluster_wait(cluster_id)
check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01'))
disks = self.fuel_web.client.get_node_disks(
self.fuel_web.get_nailgun_node_by_name('slave-01')['id'])
logger.info("Current disk partitions are: \n{d}".format(d=disks))
logger.info("Check unallocated space")
# We expect failure here only for release 5.0 due to bug
# https://bugs.launchpad.net/fuel/+bug/1306625, so it is
# necessary to assert_true in the next release.
assert_false(
checkers.check_unallocated_space(disks, contr_img_ceph=True),
"Check unallocated space on controller")
# Run ostf
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("ceph_multinode_with_cinder")
def verify_services(self):
"""Check that the correct amount of collector processes are running.
:returns: list of process IDs indexed by node and process
:rtype: dict
"""
pids = {}
processes_count = {
"collectd": 1,
"collectdmon": 1
}
if self.settings.version.startswith("0.9"):
processes_count["hekad"] = 1
else:
# Starting with 0.10, there are one collector for logs and one for
# metrics
processes_count["hekad"] = 2
online_nodes = [node for node in self.helpers.get_all_ready_nodes()
if node["online"]]
for node in online_nodes:
pids[node["name"]] = {}
with self.env.d_env.get_ssh_to_remote(node["ip"]) as remote:
for process, count in processes_count.items():
logger.info("Checking process {0} on node {1}".format(
process, node["name"]
))
pids[node["name"]][process] = (
self.checkers.check_process_count(
remote, process, count))
return pids
def setup_environment(self, custom=settings.CUSTOM_ENV,
build_images=settings.BUILD_IMAGES,
iso_connect_as=settings.ADMIN_BOOT_DEVICE,
security=settings.SECURITY_TEST):
# Create environment and start the Fuel master node
admin = self.d_env.nodes().admin
self.d_env.start([admin])
logger.info("Waiting for admin node to start up")
wait(lambda: admin.driver.node_active(admin), 60)
logger.info("Proceed with installation")
# update network parameters at boot screen
admin.send_keys(self.get_keys(admin, custom=custom,
build_images=build_images,
iso_connect_as=iso_connect_as))
if settings.SHOW_FUELMENU:
self.wait_for_fuelmenu()
else:
self.wait_for_provisioning()
self.set_admin_ssh_password()
self.wait_for_external_config()
if custom:
self.setup_customisation()
if security:
nessus_node = NessusActions(self.d_env)
nessus_node.add_nessus_node()
# wait while installation complete
self.admin_actions.modify_configs(self.d_env.router())
self.kill_wait_for_external_config()
self.wait_bootstrap()
self.admin_actions.wait_for_fuel_ready()
def upload_packages(self, local_packages_dir, centos_repo_path,
ubuntu_repo_path, clean_target=False):
logger.info("Upload fuel's packages from directory {0}."
.format(local_packages_dir))
centos_files_count = 0
ubuntu_files_count = 0
if centos_repo_path:
centos_files_count = self.ssh_manager.cond_upload(
ip=self.admin_ip,
source=local_packages_dir,
target=os.path.join(centos_repo_path, 'Packages'),
condition="(?i).*\.rpm$",
clean_target=clean_target
)
if centos_files_count > 0:
regenerate_centos_repo(centos_repo_path)
if ubuntu_repo_path:
ubuntu_files_count = self.ssh_manager.cond_upload(
ip=self.admin_ip,
source=local_packages_dir,
target=os.path.join(ubuntu_repo_path, 'pool/main'),
condition="(?i).*\.deb$",
clean_target=clean_target
)
if ubuntu_files_count > 0:
regenerate_ubuntu_repo(ubuntu_repo_path)
return centos_files_count, ubuntu_files_count
def get_oswl_services_names():
cmd = "systemctl list-units| grep oswl_ | awk '{print $1}'"
result = SSHManager().execute_on_remote(
SSHManager().admin_ip, cmd)['stdout_str'].strip()
logger.info('list of statistic services {0}'.format(
result.split('\n')))
return result.split('\n')
def build_bootstrap_image(self, **kwargs):
simple_fields = \
("ubuntu-release", "http-proxy", "https-proxy", "script",
"label", "extend-kopts", "kernel-flavor",
"root-ssh-authorized-file", "output-dir", "image-build-dir")
list_fields = ("repo", "direct-repo-addr", "package", "extra-dir")
flag_fields = ("activate", )
command = "fuel-bootstrap build "
for field in simple_fields:
if kwargs.get(field) is not None:
command += "--{0} {1} ".format(field, kwargs.get(field))
for field in list_fields:
if kwargs.get(field) is not None:
for value in kwargs.get(field):
command += "--{0} {1} ".format(field, value)
for field in flag_fields:
if kwargs.get(field) is not None:
command += "--{0} ".format(field)
logger.info("Building bootstrap image: {0}".format(command))
result = self.ssh_manager.execute_on_remote(
ip=self.admin_ip,
cmd=command,
)['stdout_str']
logger.info("Bootstrap image has been built: {0}".format(result))
uuid = self.parse_uuid(result)[0]
path = os.path.join(kwargs.get("output-dir", "/tmp"),
"{0}.tar.gz".format(uuid))
return uuid, path
def describe_environment(self):
"""Environment
:rtype : Environment
"""
environment = self.manager.environment_create(self.env_name)
networks = []
interfaces = settings.INTERFACE_ORDER
if self.multiple_cluster_networks:
logger.info('Multiple cluster networks feature is enabled!')
if settings.BONDING:
interfaces = settings.BONDING_INTERFACES.keys()
for name in interfaces:
networks.append(self.create_networks(name, environment))
for name in self.node_roles.admin_names:
self.describe_admin_node(name, networks)
for name in self.node_roles.other_names:
if self.multiple_cluster_networks:
networks1 = [net for net in networks if net.name
in settings.NODEGROUPS[0]['pools']]
networks2 = [net for net in networks if net.name
in settings.NODEGROUPS[1]['pools']]
# If slave index is even number, then attach to
# it virtual networks from the second network group.
if int(name[-2:]) % 2 == 1:
self.describe_empty_node(name, networks1)
elif int(name[-2:]) % 2 == 0:
self.describe_empty_node(name, networks2)
else:
self.describe_empty_node(name, networks)
return environment
def modify_configs(self, router):
# Slave nodes should use the gateway of 'admin' network as the default
# gateway during provisioning and as an additional DNS server.
fuel_settings = self.get_fuel_settings()
fuel_settings['DNS_UPSTREAM'] = router
fuel_settings['ADMIN_NETWORK']['dhcp_gateway'] = router
if FUEL_USE_LOCAL_NTPD:
# Try to use only ntpd on the host as the time source
# for admin node
cmd = 'ntpdate -p 4 -t 0.2 -ub {0}'.format(router)
if not self.ssh_manager.execute(ip=self.admin_ip,
cmd=cmd)['exit_code']:
# Local ntpd on the host is alive, so
# remove all NTP sources and add the host instead.
logger.info("Switching NTPD on the Fuel admin node to use "
"{0} as the time source.".format(router))
ntp_keys = [k for k in fuel_settings.keys()
if re.match(r'^NTP', k)]
for key in ntp_keys:
fuel_settings.pop(key)
fuel_settings['NTP1'] = router
if MIRROR_UBUNTU:
fuel_settings['BOOTSTRAP']['repos'] = \
replace_repos.replace_ubuntu_repos(
{
'value': fuel_settings['BOOTSTRAP']['repos']
},
upstream_host='archive.ubuntu.com')
logger.info("Replace default Ubuntu mirror URL for "
"bootstrap image in Fuel settings")
self.save_fuel_settings(fuel_settings)
def check_ceilometer_resource_functionality(self):
logger.info("Start checking Ceilometer Resource API")
fail_msg = "Failed to get resource list."
msg = "getting resources list"
resources_list = self.helpers.verify(
600, self.ceilometer_client.resources.list, 1, fail_msg, msg,
limit=10)
for resource in resources_list:
# We need to check that resource_id doesn't contain char '/'
# because if it is GET resource request fails
if "/" not in resource.resource_id:
resource_id = resource.resource_id
break
fail_msg = ("Failed to find '{}' resource with certain resource "
"ID.".format(resource_id))
msg = ("searching '{}' resource with certain resource "
"ID".format(resource_id))
self.helpers.verify(60, self.ceilometer_client.resources.get, 2,
fail_msg, msg, resource_id=resource_id)
fail_msg = "Failed to get meters list."
msg = "getting meters list"
self.helpers.verify(60, self.ceilometer_client.meters.list, 3,
fail_msg, msg, limit=10)
fail_msg = "Failed to get unique meters list."
msg = "getting unique meters list"
self.helpers.verify(60, self.ceilometer_client.meters.list, 4,
fail_msg, msg, limit=10, unique=True)
def _create_network_resources(self, tenant_id):
"""This method creates network resources.
It creates a network, an internal subnet on the network, a router and
links the network to the router. All resources created by this method
will be automatically deleted.
"""
logger.info("Creating network resources...")
net_name = "ostf-autoscaling-test-service-net"
net_body = {
"network": {
"name": net_name,
"tenant_id": tenant_id
}
}
ext_net = None
net = None
for network in self.neutron_cli.list_networks()["networks"]:
if not net and network["name"] == net_name:
net = network
if not ext_net and network["router:external"]:
ext_net = network
if not net:
net = self.neutron_cli.create_network(net_body)["network"]
subnet = self.helpers.os_conn.create_subnet(
"sub" + net_name, net["id"], "10.1.7.0/24", tenant_id=tenant_id
)
router_name = 'ostf-autoscaling-test-service-router'
router = self.helpers.os_conn.create_router(
router_name, self.helpers.os_conn.get_tenant("admin"))
self.neutron_cli.add_interface_router(
router["id"], {"subnet_id": subnet["id"]})
return net["id"]
def __init__(self, url, keystone_url, credentials, **kwargs):
logger.info('Initiate HTTPClient with url %s', url)
self.url = url
self.keystone_url = keystone_url
self.creds = dict(credentials, **kwargs)
self.keystone = None
self.opener = urllib2.build_opener(urllib2.HTTPHandler)
def post(self, endpoint, data=None, content_type="application/json"):
if not data:
data = {}
logger.info('self url is %s' % self.url)
req = urllib2.Request(self.url + endpoint, data=json.dumps(data))
req.add_header('Content-Type', content_type)
return self._open(req)
def generate_facts(ip):
ssh_manager = SSHManager()
facter_dir = '/var/lib/puppet/lib/facter'
exluded_facts = ['naily.rb']
if not ssh_manager.isdir_on_remote(ip, facter_dir):
ssh_manager.mkdir_on_remote(ip, facter_dir)
logger.debug('Directory {0} was created'.format(facter_dir))
ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir))
logger.debug('rb files were removed from {0}'.format(facter_dir))
facts_files = ssh_manager.execute_on_remote(
ip,
'find /etc/puppet/modules/ -wholename "*/lib/facter/*.rb"')['stdout']
facts_files = [i.strip() for i in facts_files]
logger.debug('The following facts {0} will'
' be copied to {1}'.format(facts_files, facter_dir))
for fact in facts_files:
if not fact or re.sub(r'.*/', '', fact) in exluded_facts:
continue
ssh_manager.execute_on_remote(ip,
'cp {0} {1}/'.format(fact, facter_dir))
logger.debug('Facts were copied')
ssh_manager.execute_on_remote(ip, 'facter -p -y > /tmp/facts.yaml')
logger.info('Facts yaml was created')
ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir))
logger.debug('rb files were removed from {0}'.format(facter_dir))
def assert_cli_task_success(
self, task, remote, timeout=70 * 60, interval=20):
logger.info('Wait {timeout} seconds for task: {task}'
.format(timeout=timeout, task=task))
start = time.time()
try:
wait(
lambda: self.get_task(
remote, task['id'])['status'] != 'running',
interval=interval,
timeout=timeout
)
except TimeoutError:
raise TimeoutError(
"Waiting timeout {timeout} sec was reached for task: {task}"
.format(task=task["name"], timeout=timeout))
took = time.time() - start
task = self.get_task(remote, task['id'])
logger.info('Task finished in {took} seconds with the result: {task}'
.format(took=took, task=task))
assert_equal(
task['status'], 'ready',
"Task '{name}' has incorrect status. {} != {}".format(
task['status'], 'ready', name=task["name"]
)
)
def replace_centos_bootstrap(environment):
"""Replaced initramfs.img in /var/www/nailgun/
with re-builded with review code
environment - Environment Model object - self.env
"""
logger.info("Updating bootstrap")
if not settings.UPDATE_FUEL:
raise Exception("{} variable don't exist"
.format(settings.UPDATE_FUEL))
try:
rebuilded_bootstrap = '/var/initramfs.img.updated'
with environment.d_env.get_admin_remote() as remote:
checkers.check_file_exists(
remote,
'{0}'.format(rebuilded_bootstrap))
logger.info("Assigning new bootstrap from {}"
.format(rebuilded_bootstrap))
bootstrap = "/var/www/nailgun/bootstrap"
cmd = ("mv {0}/initramfs.img /var/initramfs.img;"
"cp /var/initramfs.img.updated {0}/initramfs.img;"
"chmod +r {0}/initramfs.img;"
).format(bootstrap)
result = remote.execute(cmd)
assert_equal(result['exit_code'], 0,
('Failed to assign bootstrap {}'
).format(result))
cmd = "cobbler sync"
environment.base_actions.execute(cmd, exit_code=0)
except Exception as e:
logger.error("Could not update bootstrap {e}".format(e=e))
raise
def store_astute_yaml_for_one_node(nailgun_node):
ssh_manager = SSHManager()
if 'roles' not in nailgun_node:
return None
errmsg = 'Downloading "{0}.yaml" from the {1} failed'
msg = 'File "{0}.yaml" was downloaded from the {1}'
nodename = nailgun_node['name']
ip = nailgun_node['ip']
for role in nailgun_node['roles']:
filename = '{0}/{1}-{2}-{3}.yaml'.format(settings.LOGS_DIR,
func_name,
nodename,
role)
if not ssh_manager.isfile_on_remote(ip,
'/etc/{0}.yaml'.format(role)):
role = 'primary-' + role
if ssh_manager.download_from_remote(ip,
'/etc/{0}.yaml'.format(role),
filename):
logger.info(msg.format(role, nodename))
else:
logger.error(errmsg.format(role, nodename))
if settings.DOWNLOAD_FACTS:
fact_filename = re.sub(r'-\w*\.', '-facts.', filename)
generate_facts(ip)
if ssh_manager.download_from_remote(ip,
'/tmp/facts.yaml',
fact_filename):
logger.info(msg.format('facts', nodename))
else:
logger.error(errmsg.format('facts', nodename))
def deploy_one_node(self):
"""Deploy cluster with controller node only
Scenario:
1. Create cluster
2. Add 1 node with controller role
3. Deploy the cluster
4. Validate cluster was set up correctly, there are no dead
services, there are no errors in logs
Duration 20m
"""
self.env.revert_snapshot("ready")
self.fuel_web.client.list_nodes()
self.env.bootstrap_nodes(
self.env.d_env.nodes().slaves[:1])
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=DEPLOYMENT_MODE,
)
logger.info('Cluster is {!s}'.format(cluster_id))
self.fuel_web.update_nodes(
cluster_id,
{'slave-01': ['controller']}
)
self.fuel_web.deploy_cluster_wait(cluster_id)
os_conn = os_actions.OpenStackActions(
self.fuel_web.get_public_vip(cluster_id))
self.fuel_web.assert_cluster_ready(os_conn, smiles_count=4)
self.fuel_web.run_single_ostf_test(
cluster_id=cluster_id, test_sets=['sanity'],
test_name=('fuel_health.tests.sanity.test_sanity_identity'
'.SanityIdentityTest.test_list_users'))
def wrapper(*args, **kwargs):
result = func(*args, **kwargs)
try:
if settings.UPLOAD_PATCHSET:
if not settings.GERRIT_REFSPEC:
raise ValueError('REFSPEC should be set for CI tests.')
logger.info("Uploading new patchset from {0}"
.format(settings.GERRIT_REFSPEC))
remote = SSHClient(args[0].admin_node_ip,
username='******',
password='******')
remote.upload(settings.PATCH_PATH.rstrip('/'),
'/tmp/fuel-ostf')
remote.execute('source /opt/fuel_plugins/ostf/bin/activate; '
'cd /tmp/fuel-ostf; python setup.py develop')
remote.execute('/etc/init.d/supervisord restart')
helpers.wait(
lambda: "RUNNING" in
remote.execute("supervisorctl status ostf | awk\
'{print $2}'")['stdout'][0],
timeout=60)
logger.info("OSTF status: RUNNING")
except Exception as e:
logger.error("Could not upload patch set {e}".format(e=e))
raise
return result
def check_mysql(remote, node_name):
check_cmd = 'pkill -0 -x mysqld'
check_crm_cmd = ('crm resource status clone_p_mysql |'
' grep -q "is running on: $HOSTNAME"')
check_galera_cmd = ("mysql --connect_timeout=5 -sse \"SELECT"
" VARIABLE_VALUE FROM"
" information_schema.GLOBAL_STATUS"
" WHERE VARIABLE_NAME"
" = 'wsrep_local_state_comment';\"")
try:
wait(lambda: remote.execute(check_cmd)['exit_code'] == 0,
timeout=300)
logger.info('MySQL daemon is started on {0}'.format(node_name))
except TimeoutError:
logger.error('MySQL daemon is down on {0}'.format(node_name))
raise
_wait(lambda: assert_equal(remote.execute(check_crm_cmd)['exit_code'], 0,
'MySQL resource is NOT running on {0}'.format(
node_name)), timeout=60)
try:
wait(lambda: ''.join(remote.execute(
check_galera_cmd)['stdout']).rstrip() == 'Synced', timeout=600)
except TimeoutError:
logger.error('galera status is {0}'.format(''.join(remote.execute(
check_galera_cmd)['stdout']).rstrip()))
raise
def _cluster_from_config(self, config):
"""Create cluster from predefined config."""
slaves = len(config.get('nodes'))
cluster_name = config.get('name', self._context.__name__)
snapshot_name = "ready_cluster_{}".format(cluster_name)
if self.check_run(snapshot_name):
self.env.revert_snapshot(snapshot_name)
cluster_id = self.fuel_web.client.get_cluster_id(cluster_name)
self._context._storage['cluster_id'] = cluster_id
logger.info("Getted deployed cluster from snapshot")
return True
elif self.get_ready_slaves(slaves):
logger.info("Create env {}".format(cluster_name))
cluster_id = self.fuel_web.create_cluster(
name=cluster_name,
mode=config.get('mode', settings.DEPLOYMENT_MODE),
settings=config.get('settings', {})
)
self._context._storage['cluster_id'] = cluster_id
self.fuel_web.update_nodes(
cluster_id,
config.get('nodes')
)
self.fuel_web.verify_network(cluster_id)
self.fuel_web.deploy_cluster_wait(cluster_id)
self.fuel_web.verify_network(cluster_id)
self.env.make_snapshot(snapshot_name, is_make=True)
self.env.resume_environment()
return True
else:
logger.error("Can't deploy cluster because snapshot"
" with bootstrapped nodes didn't revert")
raise RuntimeError("Can't deploy cluster because snapshot"
" with bootstrapped nodes didn't revert")
def check_ipconfig_for_template(self, cluster_id, network_template,
networks):
logger.info("Checking that IP addresses configuration on nodes "
"corresponds to used networking template...")
# Network for Neutron is configured in namespaces (l3/dhcp agents)
# and a bridge for it doesn't have IP, so skipping it for now
skip_roles = {'neutron/private'}
for node in self.fuel_web.client.list_cluster_nodes(cluster_id):
node_networks = set()
node_group_name = [ng['name'] for ng in
self.fuel_web.client.get_nodegroups()
if ng['id'] == node['group_id']][0]
for role in node['roles']:
node_networks.update(
self.get_template_ep_for_role(template=network_template,
role=role,
nodegroup=node_group_name,
skip_net_roles=skip_roles))
with self.env.d_env.get_ssh_to_remote(node['ip']) as remote:
for network in networks:
if network['name'] not in node_networks or \
network['group_id'] != node['group_id']:
continue
logger.debug('Checking interface "{0}" for IP network '
'"{1}" on "{2}"'.format(network['interface'],
network['cidr'],
node['hostname']))
self.check_interface_ip_exists(remote,
network['interface'],
network['cidr'])
def prepare_repository(self):
"""Prepare admin node to packages testing
Scenario:
1. Temporary set nameserver to local router on admin node
2. Install tools to manage rpm/deb repository
3. Retrieve list of packages from custom repository
4. Download packages to local rpm/deb repository
5. Update .yaml file with new packages version
6. Re-generate repo using shell scripts on admin node
"""
# Check necessary settings and revert a snapshot
if not self.custom_pkgs_mirror:
return
logger.info("Custom mirror with new packages: {0}"
.format(settings.CUSTOM_PKGS_MIRROR))
if settings.OPENSTACK_RELEASE_UBUNTU in settings.OPENSTACK_RELEASE:
# Ubuntu
master_tools = ['dpkg', 'dpkg-devel', 'dpkg-dev']
self.install_tools(master_tools)
self.get_pkgs_list_ubuntu()
pkgs_local_path = ('{0}/pool/'
.format(self.local_mirror_ubuntu))
self.download_pkgs(pkgs_local_path)
self.regenerate_repo(self.ubuntu_script, self.local_mirror_ubuntu)
else:
# CentOS
master_tools = ['createrepo']
self.install_tools(master_tools)
self.get_pkgs_list_centos()
pkgs_local_path = '{0}/Packages/'.format(self.local_mirror_centos)
self.download_pkgs(pkgs_local_path)
self.regenerate_repo(self.centos_script, self.local_mirror_centos)
def show_step(self, step, details='', initialize=False):
"""Show a description of the step taken from docstring
:param int/str step: step number to show
:param str details: additional info for a step
"""
test_func_name = get_test_method_name()
if initialize or step == 1:
self.current_log_step = step
else:
self.current_log_step += 1
if self.current_log_step != step:
error_message = 'The step {} should be {} at {}'
error_message = error_message.format(
step,
self.current_log_step,
test_func_name
)
logger.error(error_message)
test_func = getattr(self.__class__, test_func_name)
docstring = test_func.__doc__
docstring = '\n'.join([s.strip() for s in docstring.split('\n')])
steps = {s.split('. ')[0]: s for s in
docstring.split('\n') if s and s[0].isdigit()}
if details:
details_msg = ': {0} '.format(details)
else:
details_msg = ''
if str(step) in steps:
logger.info("\n" + " " * 55 + "<<< {0} {1}>>>"
.format(steps[str(step)], details_msg))
else:
logger.info("\n" + " " * 55 + "<<< {0}. (no step description "
"in scenario) {1}>>>".format(str(step), details_msg))
def regenerate_repo(self, regenerate_script, local_mirror_path):
# Uploading scripts that prepare local repositories:
# 'regenerate_centos_repo' and 'regenerate_ubuntu_repo'
try:
self.ssh_manager.upload_to_remote(
ip=self.ip,
source='{0}/{1}'.format(self.path_scripts, regenerate_script),
target=self.remote_path_scripts
)
self.ssh_manager.execute_on_remote(
ip=self.ip,
cmd='chmod 755 {0}/{1}'.format(self.remote_path_scripts,
regenerate_script)
)
except Exception:
logger.error('Could not upload scripts for updating repositories.'
'\n{0}'.format(traceback.format_exc()))
raise
# Update the local repository using previously uploaded script.
script_cmd = '{0}/{1} {2} {3}'.format(self.remote_path_scripts,
regenerate_script,
local_mirror_path,
self.ubuntu_release)
script_result = self.ssh_manager.execute(
ip=self.ip,
cmd=script_cmd
)
assert_equal(0, script_result['exit_code'],
self.assert_msg(script_cmd, script_result['stderr']))
logger.info('Local repository {0} has been updated successfully.'
.format(local_mirror_path))
def do_sync_time(self, ntps=None):
# 0. 'ntps' can be filled by __init__() or outside the class
self.ntps = ntps or self.ntps
if not self.ntps:
raise ValueError("No servers were provided to synchronize "
"the time in self.ntps")
# 1. Set actual time on all nodes via 'ntpdate'
[ntp.set_actual_time() for ntp in self.ntps]
assert_true(self.is_synchronized, "Time on nodes was not set:"
" \n{0}".format(self.report_not_synchronized()))
# 2. Restart NTPD service
[ntp.stop() for ntp in self.ntps]
[ntp.start() for ntp in self.ntps]
# 3. Wait for established peers
[ntp.wait_peer() for ntp in self.ntps]
assert_true(self.is_connected, "Time on nodes was not synchronized:"
" \n{0}".format(self.report_not_connected()))
# 4. Report time on nodes
for ntp in self.ntps:
logger.info("Time on '{0}' = {1}".format(ntp.node_name,
ntp.date()[0].rstrip()))
def download_pkgs(self, pkgs_local_path):
# Process the packages list:
total_pkgs = len(self.pkgs_list)
logger.info('Found {0} custom package(s)'.format(total_pkgs))
for npkg, pkg in enumerate(self.pkgs_list):
# TODO: Previous versions of the updating packages must be removed
# to avoid unwanted packet manager dependencies resolution
# (when some package still depends on other package which
# is not going to be installed)
logger.info('({0}/{1}) Downloading package: {2}/{3}'
.format(npkg + 1, total_pkgs,
self.custom_pkgs_mirror,
pkg["filename:"]))
pkg_ext = pkg["filename:"].split('.')[-1]
if pkg_ext == 'deb':
path_suff = 'main/'
elif pkg_ext == 'udeb':
path_suff = 'debian-installer/'
else:
path_suff = ''
wget_cmd = "wget --no-verbose --directory-prefix {0} {1}/{2}"\
.format(pkgs_local_path + path_suff,
self.custom_pkgs_mirror,
pkg["filename:"])
wget_result = self.ssh_manager.execute(
ip=self.ip,
cmd=wget_cmd
)
assert_equal(0, wget_result['exit_code'],
self.assert_msg(wget_cmd, wget_result['stderr']))
def get_sha_sum(file_path):
logger.debug('Get md5 fo file {0}'.format(file_path))
md5_sum = SSHManager().execute_on_remote(
SSHManager().admin_ip, cmd='md5sum {0}'.format(
file_path))['stdout_str'].strip()
logger.info('MD5 is {0}'.format(md5_sum))
return md5_sum
def replace_rpm_package(package):
"""Replaced rpm package.rpm on master node with package.rpm
from review
"""
ssh = SSHManager()
logger.info("Patching {}".format(package))
if not settings.UPDATE_FUEL:
raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True')
try:
# Upload package
target_path = '/var/www/nailgun/{}/'.format(package)
ssh.upload_to_remote(
ip=ssh.admin_ip,
source=settings.UPDATE_FUEL_PATH.rstrip('/'),
target=target_path)
package_name = package
package_ext = '*.noarch.rpm'
pkg_path = os.path.join(target_path,
'{}{}'.format(package_name, package_ext))
full_package_name = get_full_filename(wildcard_name=pkg_path)
logger.debug('Package name is {0}'.format(full_package_name))
full_package_path = os.path.join(os.path.dirname(pkg_path),
full_package_name)
# Update package on master node
if not does_new_pkg_equal_to_installed_pkg(
installed_package=package_name,
new_package=full_package_path):
update_rpm(path=full_package_path)
except Exception:
logger.error("Could not upload package")
raise
def _create_net_subnet(self, cluster):
"""Create net and subnet"""
contrail_ip = self.fuel_web.get_public_vip(cluster)
logger.info('The ip is %s', contrail_ip)
net = Common(
controller_ip=contrail_ip, user='******',
password='******', tenant='admin'
)
net.neutron.create_network(body={
'network': {
'name': 'net04',
'admin_state_up': True,
}
})
network_id = ''
network_dic = net.neutron.list_networks()
for dd in network_dic['networks']:
if dd.get("name") == "net04":
network_id = dd.get("id")
if network_id == "":
logger.error('Network id empty')
logger.debug("id {0} to master node".format(network_id))
net.neutron.create_subnet(body={
'subnet': {
'network_id': network_id,
'ip_version': 4,
'cidr': '10.100.0.0/24',
'name': 'subnet04',
}
})
def update_connection(self, ip, login=None, password=None,
keys=None, port=22):
"""Update existed connection
:param ip: host ip string
:param login: login string
:param password: password string
:param keys: list of keys
:param port: ssh port int
:return: None
"""
if (ip, port) in self.connections:
logger.info('SSH_MANAGER:Close connection for {ip}:{port}'.format(
ip=ip, port=port))
self.connections[(ip, port)].clear()
logger.info('SSH_MANAGER:Create new connection for '
'{ip}:{port}'.format(ip=ip, port=port))
self.connections[(ip, port)] = SSHClient(
host=ip,
port=port,
username=login,
password=password,
private_keys=keys if keys is not None else []
)
def ceph_rados_gw(self):
"""Deploy ceph HA with RadosGW for objects
Scenario:
1. Create cluster with Neutron
2. Add 3 nodes with controller role
3. Add 3 nodes with compute and ceph-osd role
4. Deploy the cluster
5. Check ceph status
6. Run OSTF tests
7. Check the radosgw daemon is started
Duration 90m
Snapshot ceph_rados_gw
"""
def radosgw_started(remote):
return remote.check_call('pkill -0 radosgw')['exit_code'] == 0
self.env.revert_snapshot("ready")
self.env.bootstrap_nodes(
self.env.d_env.nodes().slaves[:6])
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings={
'volumes_lvm': False,
'volumes_ceph': True,
'images_ceph': True,
'objects_ceph': True,
'tenant': 'rados',
'user': '******',
'password': '******'
}
)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute', 'ceph-osd'],
'slave-05': ['compute', 'ceph-osd'],
'slave-06': ['compute', 'ceph-osd']
}
)
self.fuel_web.verify_network(cluster_id)
# Deploy cluster
self.fuel_web.deploy_cluster_wait(cluster_id)
# Network verification
self.fuel_web.verify_network(cluster_id)
# HAProxy backend checking
controller_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['controller'])
for node in controller_nodes:
logger.info("Check all HAProxy backends on {}".format(
node['meta']['system']['fqdn']))
haproxy_status = checkers.check_haproxy_backend(node['ip'])
assert_equal(haproxy_status['exit_code'], 1,
"HAProxy backends are DOWN. {0}".format(
haproxy_status))
self.fuel_web.check_ceph_status(cluster_id)
# Run ostf
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
# Check the radosgw daemon is started
with self.fuel_web.get_ssh_for_node('slave-01') as remote:
assert_true(radosgw_started(remote), 'radosgw daemon started')
self.env.make_snapshot("ceph_rados_gw")
def get_md5sum(self, file_path, controller_ssh, vm_ip, creds=()):
logger.info("Get file md5sum and compare it with previous one")
out = self.execute_through_host(
controller_ssh, vm_ip, "md5sum {:s}".format(file_path), creds)
return out['stdout']
def ceph_ha_one_controller_with_cinder(self):
"""Deploy ceph with cinder in ha mode with 1 controller
Scenario:
1. Create cluster
2. Add 1 node with controller role
3. Add 1 node with compute role
4. Add 2 nodes with cinder and ceph OSD roles
5. Deploy the cluster
6. Check ceph status
7. Check partitions on controller node
Duration 40m
Snapshot ceph_ha_one_controller_with_cinder
"""
try:
self.check_run('ceph_ha_one_controller_with_cinder')
except SkipTest:
return
self.env.revert_snapshot("ready")
self.env.bootstrap_nodes(
self.env.d_env.nodes().slaves[:4])
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings={
'volumes_ceph': False,
'images_ceph': True,
'osd_pool_size': '2',
'volumes_lvm': True,
'tenant': 'ceph2',
'user': '******',
'password': '******'
}
)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['controller'],
'slave-02': ['compute'],
'slave-03': ['cinder', 'ceph-osd'],
'slave-04': ['cinder', 'ceph-osd']
}
)
# Cluster deploy
self.fuel_web.deploy_cluster_wait(cluster_id)
self.fuel_web.check_ceph_status(cluster_id)
disks = self.fuel_web.client.get_node_disks(
self.fuel_web.get_nailgun_node_by_name('slave-01')['id'])
logger.info("Current disk partitions are: \n{d}".format(d=disks))
logger.info("Check unallocated space")
# We expect failure here only for release 5.0 due to bug
# https://bugs.launchpad.net/fuel/+bug/1306625, so it is
# necessary to assert_true in the next release.
assert_false(
checkers.check_unallocated_space(disks, contr_img_ceph=True),
"Check unallocated space on controller")
# Run ostf
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.env.make_snapshot("ceph_ha_one_controller_with_cinder",
is_make=True)
def check_rh_hard_reboot(self):
"""Check that resumed VM is working properly after hard reboot of
RH-based compute
Scenario:
1. Revert environment with RH-compute.
2. Check that services are ready.
3. Boot VM on compute and check its connectivity via floating ip.
4. Hard reboot RH-based compute.
5. Verify VM connectivity via floating ip after successful reboot
and VM resume action.
Duration 20m
Snapshot check_rh_hard_reboot
"""
self.show_step(1, initialize=True)
self.env.revert_snapshot('ready_ha_one_controller_with_rh_compute',
skip_timesync=True,
skip_slaves_check=True)
self.check_slaves_are_ready()
logger.debug('All slaves online.')
self.show_step(2)
cluster_id = self.fuel_web.get_last_created_cluster()
os_conn = os_actions.OpenStackActions(
self.fuel_web.get_public_vip(cluster_id))
self.fuel_web.assert_cluster_ready(os_conn, smiles_count=5)
logger.debug('Cluster up and ready.')
self.show_step(3)
cluster_id = self.fuel_web.get_last_created_cluster()
controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, roles=('controller', ))
os_conn = os_actions.OpenStackActions(
self.fuel_web.get_public_vip(cluster_id))
asserts.assert_equal(
len(controllers), 1,
'Environment does not have 1 controller node, '
'found {} nodes!'.format(len(controllers)))
compute = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['compute'])[0]
target_node = self.fuel_web.get_devops_node_by_nailgun_node(compute)
target_node_ip = self.fuel_web.get_node_ip_by_devops_name(
target_node.name)
net_label = self.fuel_web.get_cluster_predefined_networks_name(
cluster_id)['private_net']
vm = os_conn.create_server_for_migration(neutron=True, label=net_label)
vm_floating_ip = os_conn.assign_floating_ip(vm)
logger.info('Trying to get vm via tcp.')
try:
wait(lambda: tcp_ping(vm_floating_ip.ip, 22), timeout=120)
except TimeoutError:
raise TimeoutError('Can not ping instance'
' by floating ip {0}'.format(vm_floating_ip.ip))
logger.info('VM is accessible via ip: {0}'.format(vm_floating_ip.ip))
self.show_step(4)
target_node.destroy()
asserts.assert_false(target_node.driver.node_active(node=target_node),
'Target node still active')
target_node.start()
asserts.assert_true(target_node.driver.node_active(node=target_node),
'Target node did not start')
self.wait_for_slave_provision(target_node_ip)
self.fuel_web.assert_cluster_ready(os_conn, smiles_count=5)
logger.info('All cluster services up and '
'running after compute hard reboot.')
self.show_step(5)
asserts.assert_equal(
os_conn.get_instance_detail(vm).status, "ACTIVE",
"Instance did not reach active state after compute back online, "
"current state is {0}".format(
os_conn.get_instance_detail(vm).status))
logger.info('Spawned VM is ACTIVE. Trying to '
'access it via ip: {0}'.format(vm_floating_ip.ip))
try:
wait(lambda: tcp_ping(vm_floating_ip.ip, 22), timeout=120)
except TimeoutError:
raise TimeoutError('Can not ping instance'
' by floating ip {0}'.format(vm_floating_ip.ip))
logger.info('VM is accessible. Deleting it.')
os_conn.delete_instance(vm)
os_conn.verify_srv_deleted(vm)
def admin_install_updates(self):
"""Update packages using yum and install updates via
update-master-node.sh tool"""
logger.info('Searching for updates..')
update_command = 'yum clean expire-cache && ' \
'yum update -y 2>>/var/log/yum-update-error.log'
logger.info('Performing yum clean and update commands')
update_result = self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command=update_command,
error_info='Packages update failed, inspect logs for details')
logger.info('Packages were updated successfully')
# Check if any packets were updated and update was successful
match_updated_count = re.search(r'Upgrade\s+(\d+)\s+Package',
update_result['stdout_str'])
# In case of package replacement, the new one is marked as
# installed and the old one as removed
match_installed_count = re.search(r'Install\s+(\d+)\s+Package',
update_result['stdout_str'])
match_complete_message = re.search(r'Complete!',
update_result['stdout_str'])
match_no_updates = re.search("No Packages marked for Update",
update_result['stdout_str'])
if match_no_updates or not match_complete_message \
or not (match_updated_count or match_installed_count):
logger.warning('No updates were found or update was incomplete.')
return
updates_count = 0
if match_updated_count:
updates_count += int(match_updated_count.group(1))
if match_installed_count:
updates_count += int(match_installed_count.group(1))
logger.info('{0} package(s) were updated'.format(updates_count))
logger.info('Applying updates via update-master-node.sh')
# LP #1664635 - we need to redirect stdout to /dev/null to avoid
# ssh connection hanging on massive output from puppet run.
cmd = '/usr/share/fuel-utils/update-master-node.sh > /dev/null 2>&1'
self.ssh_manager.check_call(
ip=self.ssh_manager.admin_ip,
command=cmd,
error_info='Update failed, inspect logs for details',
)
logger.info('Update successful')
def setup_customisation(self):
logger.info('Installing custom packages/manifests '
'before master node bootstrap...')
def contrail_ha_with_shutdown_contrail_node(self):
"""Verify HA with deleting Contrail roles
Scenario:
1. Create an environment with
"Neutron with tunneling segmentation"
as a network configuration
2. Enable and configure Contrail plugin
3. Add some controller, compute and storage nodes
4. Add 4 nodes with "contrail-db", "contarail-config" and
"contrail-control" roles
5. Deploy cluster
6. Run OSTF tests
7. Check Controller and Contrail nodes status
8. Shutdown node with 'contrail-db', "contarail-config" and
"contrail-control" roles
9. Deploy changes
10. Run OSTF tests
11. Check Controller and Contrail nodes status
"""
plugin.prepare_contrail_plugin(self, slaves=9)
plugin.activate_plugin(self) # enable plugin in contrail settings
vsrx_setup_result = plugin.activate_vsrx() # activate vSRX image
conf_no_contrail = {
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute'],
'slave-05': ['cinder'],
# Here slave-06 with contrail
'slave-07': ['contrail-db', 'contrail-config', 'contrail-control'],
'slave-08': ['contrail-db', 'contrail-config', 'contrail-control'],
'slave-09': ['contrail-db', 'contrail-config', 'contrail-control'],
}
conf_contrail = {
'slave-06': ['contrail-db', 'contrail-config', 'contrail-control']
}
def check_node_state(cluster_id, node_name, node_state):
"""Checks node state by it's name"""
for node in self.fuel_web.client.list_cluster_nodes(cluster_id):
if node_name in node['name']:
assert_equal(
node['status'], node_state,
'Nailgun node status is not %s but %s' %
(node_state, node['status']))
# Deploy cluster and run OSTF
openstack.update_deploy_check(self,
dict(conf_no_contrail, **conf_contrail),
is_vsrx=vsrx_setup_result)
# Check all nodes are 'ready'
for node_name in dict(conf_no_contrail, **conf_contrail):
check_node_state(self.cluster_id, node_name, 'ready')
# Shutdown contrail node
for node in self.fuel_web.client.list_cluster_nodes(self.cluster_id):
if 'slave-06' in node['name']:
logger.info('Shutdown node "%s"' % node['name'])
self.fuel_web.warm_shutdown_nodes(
self.fuel_web.get_devops_nodes_by_nailgun_nodes([node]))
break
# Run OSTF tests again
if vsrx_setup_result:
self.fuel_web.run_ostf(cluster_id=self.cluster_id)
# Check controller and contrail nodes states
node_roles = {'controller', 'contrail-config'}
for node_name, roles in conf_no_contrail.items():
if node_roles & set(roles):
check_node_state(self.cluster_id, node_name, 'ready')
def create_image(self, **kwargs):
image = self.glance.images.create(**kwargs)
logger.info("Created image: '{0}'".format(image.id))
logger.info("Image status: '{0}'".format(image.status))
return image
def dvs_regression(self):
"""Deploy cluster with plugin and vmware datastore backend.
Scenario:
1. Revert to dvs_bvt snapshot.
2. Create non default network net_1.
3. Launch instances with created network in nova and vcenter az.
4. Create Security groups.
5. Attached created security groups to instances.
6. Check connection between instances from different az.
Duration: 1.8 hours
"""
self.show_step(1)
self.env.revert_snapshot("dvs_bvt")
cluster_id = self.fuel_web.get_last_created_cluster()
os_ip = self.fuel_web.get_public_vip(cluster_id)
os_conn = os_actions.OpenStackActions(os_ip, SERVTEST_USERNAME,
SERVTEST_PASSWORD,
SERVTEST_TENANT)
tenant = os_conn.get_tenant(SERVTEST_TENANT)
# Create non default network with subnet
self.show_step(2)
logger.info('Create network {}'.format(self.net_data[0].keys()[0]))
net_1 = os_conn.create_network(network_name=self.net_data[0].keys()[0],
tenant_id=tenant.id)['network']
subnet = os_conn.create_subnet(
subnet_name=net_1['name'],
network_id=net_1['id'],
cidr=self.net_data[0][self.net_data[0].keys()[0]],
ip_version=4)
# Check that network are created
assert_true(os_conn.get_network(net_1['name'])['id'] == net_1['id'])
# Add net_1 to default router
router = os_conn.get_router(os_conn.get_network(self.ext_net_name))
os_conn.add_router_interface(router_id=router["id"],
subnet_id=subnet["id"])
self.show_step(3)
# Launch 2 vcenter VMs and 2 nova VMs in the tenant network net_01
openstack.create_instances(os_conn=os_conn,
vm_count=1,
nics=[{
'net-id': net_1['id']
}])
# Launch 2 vcenter VMs and 2 nova VMs in the default network
net_1 = os_conn.nova.networks.find(label=self.inter_net_name)
instances = openstack.create_instances(os_conn=os_conn,
vm_count=1,
nics=[{
'net-id': net_1.id
}])
openstack.verify_instance_state(os_conn)
self.show_step(4)
# Create security groups SG_1 to allow ICMP traffic.
# Add Ingress rule for ICMP protocol to SG_1
# Create security groups SG_2 to allow TCP traffic 22 port.
# Add Ingress rule for TCP protocol to SG_2
sec_name = ['SG1', 'SG2']
sg1 = os_conn.nova.security_groups.create(sec_name[0], "descr")
sg2 = os_conn.nova.security_groups.create(sec_name[1], "descr")
rulesets = [
{
# ssh
'ip_protocol': 'tcp',
'from_port': 22,
'to_port': 22,
'cidr': '0.0.0.0/0',
},
{
# ping
'ip_protocol': 'icmp',
'from_port': -1,
'to_port': -1,
'cidr': '0.0.0.0/0',
}
]
os_conn.nova.security_group_rules.create(sg1.id, **rulesets[0])
os_conn.nova.security_group_rules.create(sg2.id, **rulesets[1])
# Remove default security group and attach SG_1 and SG2 to VMs
self.show_step(5)
srv_list = os_conn.get_servers()
for srv in srv_list:
srv.remove_security_group(srv.security_groups[0]['name'])
srv.add_security_group(sg1.id)
srv.add_security_group(sg2.id)
fip = openstack.create_and_assign_floating_ips(os_conn, instances)
# Check ping between VMs
self.show_step(6)
ip_pair = dict.fromkeys(fip)
for key in ip_pair:
ip_pair[key] = [value for value in fip if key != value]
openstack.check_connection_vms(ip_pair)
def migrate_vm_backed_with_ceph(self):
"""Check VM backed with ceph migration in ha mode with 1 controller
Scenario:
1. Create cluster
2. Add 1 node with controller and ceph OSD roles
3. Add 2 nodes with compute and ceph OSD roles
4. Deploy the cluster
5. Check ceph status
6. Run OSTF
7. Create a new VM, assign floating ip
8. Migrate VM
9. Check cluster and server state after migration
10. Terminate VM
11. Check that DHCP lease is not offered for MAC of deleted VM
12. Create a new VM for migration, assign floating ip
13. Create a volume and attach it to the VM
14. Create filesystem on the new volume and mount it to the VM
15. Migrate VM
16. Mount the volume after migration
17. Check cluster and server state after migration
18. Terminate VM
Duration 35m
Snapshot vm_backed_with_ceph_live_migration
"""
self.env.revert_snapshot("ready_with_3_slaves")
self.show_step(1)
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings={
'volumes_ceph': True,
'images_ceph': True,
'ephemeral_ceph': True,
'volumes_lvm': False,
'net_provider': 'neutron',
'net_segment_type': NEUTRON_SEGMENT_TYPE,
})
self.show_step(2)
self.show_step(3)
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': ['controller', 'ceph-osd'],
'slave-02': ['compute', 'ceph-osd'],
'slave-03': ['compute', 'ceph-osd']
})
creds = ("cirros", "test")
self.show_step(4)
# Cluster deploy
self.fuel_web.deploy_cluster_wait(cluster_id)
def _check():
# Run volume test several times with hope that it pass
test_path = map_ostf.OSTF_TEST_MAPPING.get(
'Create volume and attach it to instance')
logger.debug('Start to run test {0}'.format(test_path))
self.fuel_web.run_single_ostf_test(cluster_id,
test_sets=['smoke'],
test_name=test_path)
self.show_step(5)
try:
_check()
except AssertionError:
logger.debug(AssertionError)
logger.debug("Test failed from first probe,"
" we sleep 60 second try one more time "
"and if it fails again - test will fails ")
time.sleep(60)
_check()
self.show_step(6)
# Run ostf
self.fuel_web.run_ostf(cluster_id)
self.show_step(7)
# Create new server
os = os_actions.OpenStackActions(
self.fuel_web.get_public_vip(cluster_id))
logger.info("Create new server")
srv = os.create_server_for_migration(
neutron=True,
scenario='./fuelweb_test/helpers/instance_initial_scenario')
logger.info("Srv is currently in status: %s" % srv.status)
# Prepare to DHCP leases checks
srv_instance_ip = os.get_nova_instance_ip(srv, net_name='net04')
srv_host_name = self.fuel_web.find_devops_node_by_nailgun_fqdn(
os.get_srv_hypervisor_name(srv),
self.env.d_env.nodes().slaves[:3]).name
net_id = os.get_network('net04')['id']
ports = os.get_neutron_dhcp_ports(net_id)
dhcp_server_ip = ports[0]['fixed_ips'][0]['ip_address']
with self.fuel_web.get_ssh_for_node(srv_host_name) as srv_remote_node:
srv_instance_mac = os.get_instance_mac(srv_remote_node, srv)
logger.info("Assigning floating ip to server")
floating_ip = os.assign_floating_ip(srv)
srv_host = os.get_srv_host_name(srv)
logger.info("Server is on host %s" % srv_host)
wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120)
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
md5before = os.get_md5sum("/home/test_file", remote,
floating_ip.ip, creds)
self.show_step(8)
logger.info("Get available computes")
avail_hosts = os.get_hosts_for_migr(srv_host)
logger.info("Migrating server")
new_srv = os.migrate_server(srv, avail_hosts[0], timeout=200)
logger.info("Check cluster and server state after migration")
wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120)
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
md5after = os.get_md5sum("/home/test_file", remote, floating_ip.ip,
creds)
assert_true(
md5after in md5before, "Md5 checksums don`t match."
"Before migration md5 was equal to: {bef}"
"Now it eqals: {aft}".format(bef=md5before, aft=md5after))
self.show_step(9)
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
res = os.execute_through_host(
remote, floating_ip.ip,
"ping -q -c3 -w10 {0} | grep 'received' |"
" grep -v '0 packets received'".format(
settings.PUBLIC_TEST_IP), creds)
logger.info("Ping {0} result on vm is: {1}".format(
settings.PUBLIC_TEST_IP, res['stdout']))
logger.info("Check Ceph health is ok after migration")
self.fuel_web.check_ceph_status(cluster_id)
logger.info("Server is now on host %s" % os.get_srv_host_name(new_srv))
self.show_step(10)
logger.info("Terminate migrated server")
os.delete_instance(new_srv)
assert_true(os.verify_srv_deleted(new_srv),
"Verify server was deleted")
self.show_step(11)
# Check if the dhcp lease for instance still remains
# on the previous compute node. Related Bug: #1391010
with self.fuel_web.get_ssh_for_node('slave-01') as remote:
dhcp_port_tag = ovs_get_tag_by_port(remote, ports[0]['id'])
assert_false(
checkers.check_neutron_dhcp_lease(remote, srv_instance_ip,
srv_instance_mac,
dhcp_server_ip,
dhcp_port_tag),
"Instance has been deleted, but it's DHCP lease "
"for IP:{0} with MAC:{1} still offers by Neutron DHCP"
" agent.".format(srv_instance_ip, srv_instance_mac))
self.show_step(12)
# Create a new server
logger.info("Create a new server for migration with volume")
srv = os.create_server_for_migration(
neutron=True,
scenario='./fuelweb_test/helpers/instance_initial_scenario')
logger.info("Srv is currently in status: %s" % srv.status)
logger.info("Assigning floating ip to server")
floating_ip = os.assign_floating_ip(srv)
srv_host = os.get_srv_host_name(srv)
logger.info("Server is on host %s" % srv_host)
self.show_step(13)
logger.info("Create volume")
vol = os.create_volume()
logger.info("Attach volume to server")
os.attach_volume(vol, srv)
self.show_step(14)
wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120)
logger.info("Create filesystem and mount volume")
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
os.execute_through_host(remote, floating_ip.ip,
'sudo sh /home/mount_volume.sh', creds)
os.execute_through_host(remote, floating_ip.ip,
'sudo touch /mnt/file-on-volume', creds)
self.show_step(15)
logger.info("Get available computes")
avail_hosts = os.get_hosts_for_migr(srv_host)
logger.info("Migrating server")
new_srv = os.migrate_server(srv, avail_hosts[0], timeout=120)
logger.info("Check cluster and server state after migration")
wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120)
self.show_step(16)
logger.info("Mount volume after migration")
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
out = os.execute_through_host(remote, floating_ip.ip,
'sudo mount /dev/vdb /mnt', creds)
logger.info("out of mounting volume is: %s" % out['stdout'])
with self.fuel_web.get_ssh_for_node("slave-01") as remote:
out = os.execute_through_host(remote, floating_ip.ip,
"sudo ls /mnt", creds)
assert_true("file-on-volume" in out['stdout'],
"File is abscent in /mnt")
self.show_step(17)
logger.info("Check Ceph health is ok after migration")
self.fuel_web.check_ceph_status(cluster_id)
logger.info("Server is now on host %s" % os.get_srv_host_name(new_srv))
self.show_step(18)
logger.info("Terminate migrated server")
os.delete_instance(new_srv)
assert_true(os.verify_srv_deleted(new_srv),
"Verify server was deleted")
self.env.make_snapshot("vm_backed_with_ceph_live_migration")
def remove_controllers(self):
"""Deploy cluster with 3 controllers, remove 2 controllers
and re-deploy, check hosts and corosync
Scenario:
1. Create cluster
2. Add 3 controller, 1 compute
3. Deploy the cluster
4. Remove 2 controllers
5. Deploy changes
6. Run OSTF
7. Verify networks
8. Check /etc/hosts that removed nodes aren't present
9. Check corosync.conf that removed nodes aren't present
Duration 120m
Snapshot remove_controllers
"""
self.env.revert_snapshot("ha_scale_group_2_cluster")
self.show_step(1, initialize=True)
self.show_step(2)
self.show_step(3)
cluster_id = self.fuel_web.get_last_created_cluster()
hosts = []
for node_name in ('slave-02', 'slave-03'):
node = self.fuel_web.get_nailgun_node_by_devops_node(
self.env.d_env.get_node(name=node_name))
hostname = ''.join(
self.ssh_manager.check_call(
ip=node['ip'], command="hostname")['stdout']).strip()
hosts.append(hostname)
logger.debug('hostname are {}'.format(hosts))
nodes = {'slave-02': ['controller'], 'slave-03': ['controller']}
self.show_step(4)
self.fuel_web.update_nodes(cluster_id, nodes, False, True)
self.show_step(5)
self.fuel_web.deploy_cluster_wait(cluster_id)
self.show_step(6)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.show_step(7)
self.fuel_web.verify_network(cluster_id)
node = self.fuel_web.get_nailgun_node_by_devops_node(
self.env.d_env.get_node(name='slave-01'))
for host in hosts:
self.show_step(8, initialize=True)
cmd = "grep '{}' /etc/hosts".format(host)
logger.info('Checking hosts on {}'.format(host))
result = self.ssh_manager.check_call(ip=node['ip'],
command=cmd,
expected=[1])
assert_equal(result['exit_code'], 1,
"host {} is present in /etc/hosts".format(host))
self.show_step(9)
cmd = "grep '{}' /etc/corosync/corosync.conf".format(host)
logger.info('Checking corosync.conf on {}'.format(host))
result = self.ssh_manager.check_call(ip=node['ip'],
command=cmd,
expected=[1])
assert_equal(
result['exit_code'], 1, "host {} is present in"
" /etc/corosync/corosync.conf".format(host))
self.env.make_snapshot("remove_controllers")
def check_ceph_partitions_after_reboot(self):
"""Check that Ceph OSD partitions are remounted after reboot
Scenario:
1. Create cluster in Ha mode with 1 controller
2. Add 1 node with controller role
3. Add 1 node with compute and Ceph OSD roles
4. Add 1 node with Ceph OSD role
5. Deploy the cluster
6. Check Ceph status
7. Read current partitions
8. Warm-reboot Ceph nodes
9. Read partitions again
10. Check Ceph health
11. Cold-reboot Ceph nodes
12. Read partitions again
13. Check Ceph health
Duration 40m
Snapshot check_ceph_partitions_after_reboot
"""
self.env.revert_snapshot("ready_with_3_slaves")
self.show_step(1)
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings={
'volumes_ceph': True,
'images_ceph': True,
'ephemeral_ceph': True,
'volumes_lvm': False,
'net_provider': 'neutron',
'net_segment_type': NEUTRON_SEGMENT_TYPE,
})
self.show_step(2)
self.show_step(3)
self.show_step(4)
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': ['controller'],
'slave-02': ['compute', 'ceph-osd'],
'slave-03': ['ceph-osd']
})
self.show_step(5)
# Deploy cluster
self.fuel_web.deploy_cluster_wait(cluster_id)
self.show_step(6)
for node in ["slave-02", "slave-03"]:
self.show_step(7, node)
logger.info("Get partitions for {node}".format(node=node))
_ip = self.fuel_web.get_nailgun_node_by_name(node)['ip']
with self.env.d_env.get_ssh_to_remote(_ip) as remote:
before_reboot_partitions = [
checkers.get_ceph_partitions(remote,
"/dev/vd{p}".format(p=part))
for part in ["b", "c"]
]
self.show_step(8, node)
logger.info("Warm-restart nodes")
self.fuel_web.warm_restart_nodes(
[self.fuel_web.environment.d_env.get_node(name=node)])
self.show_step(9, node)
logger.info(
"Get partitions for {node} once again".format(node=node))
_ip = self.fuel_web.get_nailgun_node_by_name(node)['ip']
with self.env.d_env.get_ssh_to_remote(_ip) as remote:
after_reboot_partitions = [
checkers.get_ceph_partitions(remote,
"/dev/vd{p}".format(p=part))
for part in ["b", "c"]
]
if before_reboot_partitions != after_reboot_partitions:
logger.info("Partitions don`t match")
logger.info("Before reboot: %s" % before_reboot_partitions)
logger.info("After reboot: %s" % after_reboot_partitions)
raise Exception()
self.show_step(10, node)
logger.info("Check Ceph health is ok after reboot")
self.fuel_web.check_ceph_status(cluster_id)
self.show_step(11, node)
logger.info("Cold-restart nodes")
self.fuel_web.cold_restart_nodes(
[self.fuel_web.environment.d_env.get_node(name=node)])
self.show_step(12, node)
_ip = self.fuel_web.get_nailgun_node_by_name(node)['ip']
with self.env.d_env.get_ssh_to_remote(_ip) as remote:
after_reboot_partitions = [
checkers.get_ceph_partitions(remote,
"/dev/vd{p}".format(p=part))
for part in ["b", "c"]
]
if before_reboot_partitions != after_reboot_partitions:
logger.info("Partitions don`t match")
logger.info("Before reboot: %s" % before_reboot_partitions)
logger.info("After reboot: %s" % after_reboot_partitions)
raise Exception()
self.show_step(13, node)
logger.info("Check Ceph health is ok after reboot")
self.fuel_web.check_ceph_status(cluster_id)
def delete_custom_nodegroup(self):
"""Delete nodegroup, check its nodes are marked as 'error'
Scenario:
1. Revert snapshot with cluster with nodes in custom nodegroup
2. Save cluster network configuration
3. Reset cluster
4. Remove custom nodegroup
5. Check nodes from custom nodegroup have 'error' status
6. Re-create custom nodegroup and upload saved network configuration
7. Assign 'error' nodes to new nodegroup
8. Check nodes from custom nodegroup are in 'discover' state
Duration 30m
"""
self.show_step(1, initialize=True)
self.env.revert_snapshot('deploy_controllers_from_custom_nodegroup')
cluster_id = self.fuel_web.get_last_created_cluster()
self.fuel_web.assert_nodes_in_ready_state(cluster_id)
self.show_step(2)
network_config = self.fuel_web.client.get_networks(cluster_id)
self.show_step(3)
custom_nodes = self.env.d_env.nodes().slaves[3:6]
self.fuel_web.stop_reset_env_wait(cluster_id)
# TODO(apanchenko): remove sleep(181) workaround when the issue with
# TODO(apanchenko): cluster reset is fixed (see LP#1588193)
# Nailgun waits 180 seconds before marking slave node as offline
time.sleep(181)
logger.info('Waiting for all nodes online for 900 seconds...')
wait(lambda: all(n['online'] for n in self.fuel_web.client.
list_cluster_nodes(cluster_id)),
timeout=15 * 60,
timeout_msg='Timeout while waiting nodes to become online '
'after reset')
self.show_step(4)
custom_nodegroup = [
ng for ng in self.fuel_web.client.get_nodegroups()
if ng['name'] == NODEGROUPS[1]['name']
][0]
self.fuel_web.client.delete_nodegroup(custom_nodegroup['id'])
self.show_step(5)
logger.info('Wait all nodes from custom nodegroup become '
'in error state..')
for slave in custom_nodes:
# pylint: disable=undefined-loop-variable
wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(slave)[
'status'] == 'error',
timeout=60 * 5,
timeout_msg='Node {} status wasn\'t changed '
'to "error"!'.format(slave.name))
# pylint: enable=undefined-loop-variable
logger.info('Node {} is in "error" state'.format(slave.name))
self.show_step(6)
new_nodegroup = self.fuel_web.client.create_nodegroup(
cluster_id, NODEGROUPS[1]['name'])
logger.debug('Updating custom nodegroup ID in network configuration..')
network_config_new = self.fuel_web.client.get_networks(cluster_id)
for network in network_config['networks']:
if network['group_id'] == custom_nodegroup['id']:
network['group_id'] = new_nodegroup['id']
for new_network in network_config_new['networks']:
if new_network['name'] == network['name'] and \
new_network['group_id'] == network['group_id']:
network['id'] = new_network['id']
self.fuel_web.client.update_network(
cluster_id, network_config['networking_parameters'],
network_config['networks'])
self.show_step(7)
self.fuel_web.client.assign_nodegroup(new_nodegroup['id'], [
self.fuel_web.get_nailgun_node_by_devops_node(node)
for node in custom_nodes
])
self.show_step(8)
logger.info('Wait all nodes from custom nodegroup become '
'in discover state..')
for slave in custom_nodes:
wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(slave)[
'status'] == 'discover',
timeout=60 * 5,
timeout_msg='Node {} status wasn\'t changed '
'to "discover"!'.format(slave.name))
logger.info('Node {} is in "discover" state'.format(slave.name))
self.env.make_snapshot("delete_custom_nodegroup")
def compute_stop_reinstallation(self):
"""Verify stop reinstallation of compute.
Scenario:
1. Revert the snapshot
2. Create an OS volume and OS instance
3. Mark 'cinder' and 'vm' partitions to be preserved
4. Stop reinstallation process of compute
5. Start the reinstallation process again
6. Run network verification
7. Run OSTF
8. Verify that the volume is present and has 'available' status
after the node reinstallation
9. Verify that the VM is available and pingable
after the node reinstallation
Duration: 115m
"""
self.env.revert_snapshot("node_reinstallation_env")
cluster_id = self.fuel_web.get_last_created_cluster()
# Create an OS volume
os_conn = os_actions.OpenStackActions(
self.fuel_web.get_public_vip(cluster_id))
volume = os_conn.create_volume()
# Create an OS instance
cmp_host = os_conn.get_hypervisors()[0]
net_label = self.fuel_web.get_cluster_predefined_networks_name(
cluster_id)['private_net']
vm = os_conn.create_server_for_migration(
neutron=True,
availability_zone="nova:{0}".format(cmp_host.hypervisor_hostname),
label=net_label)
vm_floating_ip = os_conn.assign_floating_ip(vm)
devops_helpers.wait(
lambda: devops_helpers.tcp_ping(vm_floating_ip.ip, 22),
timeout=120)
cmp_nailgun = self.fuel_web.get_nailgun_node_by_fqdn(
cmp_host.hypervisor_hostname)
# Mark 'cinder' and 'vm' partitions to be preserved
with self.env.d_env.get_admin_remote() as remote:
preserve_partition(remote, cmp_nailgun['id'], "cinder")
preserve_partition(remote, cmp_nailgun['id'], "vm")
slave_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id)
devops_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes(
slave_nodes)
logger.info('Stop reinstallation process')
self._stop_reinstallation(self.fuel_web, cluster_id,
[str(cmp_nailgun['id'])], devops_nodes)
self.fuel_web.verify_network(cluster_id)
logger.info('Start the reinstallation process again')
NodeReinstallationEnv.reinstall_nodes(self.fuel_web, cluster_id,
[str(cmp_nailgun['id'])])
self.fuel_web.verify_network(cluster_id)
self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity'])
# Verify that the created volume is still available
try:
volume = os_conn.cinder.volumes.get(volume.id)
except NotFound:
raise AssertionError(
"{0} volume is not available after its {1} hosting node "
"reinstallation".format(volume.id, cmp_nailgun['fqdn']))
expected_status = "available"
assert_equal(
expected_status, volume.status,
"{0} volume status is {1} after its {2} hosting node "
"reinstallation. Expected status is {3}.".format(
volume.id, volume.status, cmp_nailgun['fqdn'],
expected_status))
# Verify that the VM is still available
try:
os_conn.verify_instance_status(vm, 'ACTIVE')
except AssertionError:
raise AssertionError(
"{0} VM is not available after its {1} hosting node "
"reinstallation".format(vm.name, cmp_host.hypervisor_hostname))
assert_true(
devops_helpers.tcp_ping(vm_floating_ip.ip, 22),
"{0} VM is not accessible via its {1} floating "
"ip".format(vm.name, vm_floating_ip))
def deploy_controllers_from_custom_nodegroup(self):
"""Assigning controllers to non-default nodegroup
Scenario:
1. Revert snapshot with ready master node
2. Create environment with Neutron VXLAN and custom nodegroup
3. Configure network floating ranges to use public network
from custom nodegroup
4. Bootstrap slaves from custom nodegroup
5. Bootstrap slave nodes from default nodegroup
6. Add 3 nodes from 'custom' nodegroup as controllers
Add 2 nodes from 'default' nodegroup as compute and cinder
7. Run network verification
8. Deploy environment
9. Run network verification
10. Run OSTF
11. Check addresses allocated for VIPs belong to networks
from custom nodegroup
Duration 120m
Snapshot deploy_controllers_from_custom_nodegroup
"""
if not MULTIPLE_NETWORKS:
raise SkipTest('MULTIPLE_NETWORKS not enabled')
self.show_step(1, initialize=True)
self.check_run("deploy_controllers_from_custom_nodegroup")
self.env.revert_snapshot("ready")
self.show_step(2)
cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__,
mode=DEPLOYMENT_MODE_HA,
settings={
"net_provider":
'neutron',
"net_segment_type":
NEUTRON_SEGMENT['tun']
},
configure_ssl=False)
self.show_step(3)
# floating range
public2_cidr = self.env.d_env.get_network(name='public2').ip
new_settings_float = {
'floating_ranges': [[
str(public2_cidr[len(public2_cidr) // 2]),
str(public2_cidr[-2])
]]
}
self.fuel_web.client.update_network(cluster_id, new_settings_float)
self.show_step(4)
custom_nodes = self.env.d_env.nodes().slaves[3:6]
self.env.bootstrap_nodes(custom_nodes) # nodes 4, 5 and 6
self.show_step(5)
default_nodes = self.env.d_env.nodes().slaves[0:2]
self.env.bootstrap_nodes(default_nodes) # nodes 1 and 2
self.show_step(6)
default_nodegroup = NODEGROUPS[0]['name']
custom_nodegroup = NODEGROUPS[1]['name']
self.fuel_web.update_nodes(
cluster_id, {
'slave-04': [['controller'], custom_nodegroup],
'slave-05': [['controller'], custom_nodegroup],
'slave-06': [['controller'], custom_nodegroup],
'slave-01': [['compute'], default_nodegroup],
'slave-02': [['cinder'], default_nodegroup]
})
# configuring ssl after nodes added to cluster due to vips in custom ng
self.fuel_web.ssl_configure(cluster_id)
self.show_step(7)
self.fuel_web.verify_network(cluster_id)
self.show_step(8)
self.fuel_web.deploy_cluster_wait(cluster_id, timeout=150 * 60)
self.show_step(9)
self.fuel_web.verify_network(cluster_id)
self.show_step(10)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.show_step(11)
current_settings = self.fuel_web.client.get_networks(cluster_id)
check = {
'vrouter_pub': 'public2',
'management': 'management2',
'public': 'public2',
'vrouter': 'management2'
}
for k in check:
vip = netaddr.IPAddress(str(current_settings['vips'][k]['ipaddr']))
custom_net = netaddr.IPNetwork(
str(self.env.d_env.get_network(name=check[k]).ip))
asserts.assert_true(
vip in custom_net,
'{0} is not from {1} network'.format(k, check[k]))
logger.info('{0} is from {1} network'.format(k, check[k]))
self.env.make_snapshot("deploy_controllers_from_custom_nodegroup",
is_make=True)
def deploy_with_redeploy_and_modify_settings(self):
"""Deploy iteratively clusters from config, modify settings, redeploy
Scenario:
1. Load clusters' configurations from the file
2. Revert snapshot with appropriate nodes count
3. Create a cluster from config
4. Update nodes accordingly to the config
5. Deploy the cluster
6. Run OSTF
7. Get cluster attributes
8. Modify randomly cluster attributes
9. Add if it's needed ceph nodes
10. Update cluster attributes with changed one
11. Redeploy cluster
12. Run OSTF
13. Go to the next config
Duration xxx m
Snapshot will be made for all failed configurations
"""
fail_trigger = False
failed_confs = []
self.show_step(1)
for conf in self.load_config('cluster_configs.yaml'):
logger.info("Creating cluster from config with name: {}".format(
conf['name']))
self.show_step(2, details=conf['name'], initialize=True)
self.revert_snapshot(len(conf['nodes']))
self.show_step(3, details=conf['name'])
self.create_cluster(conf)
self.show_step(4, details=conf['name'])
self.update_nodes(conf)
self.show_step(5, details=conf['name'])
if not self.deploy_cluster():
logger.error("Initial deployment of cluster {0} "
"with config name {1} was failed. "
"Go to the next config".format(
self.cluster_name, conf['name']))
fail_trigger = True
failed_confs.append(conf['name'])
continue
self.show_step(6, details=conf['name'])
if not self.run_ostf():
fail_trigger = True
failed_confs.append(conf['name'])
logger.error(
"Failed to pass OSTF tests for first time deployed "
"cluster with config {}".format(conf['name']))
continue
self.show_step(7, details=conf['name'])
attrs = self.get_cluster_attributes()
self.show_step(8, details=conf['name'])
changer = SettingsChanger(attrs)
logger.info("The options below will NOT be changed: {}".format(
changer.SKIPPED_FIELDS_LIST))
changer.make_changes(options=None, randomize=30)
new_attrs = changer.attrs
self.show_step(9, details=conf['name'])
ceph_nodes_count = self.check_config_for_ceph(new_attrs)
existed_ceph_count = self.get_existed_ceph_nodes_count(conf)
if ceph_nodes_count > existed_ceph_count:
count = len(conf['nodes'])
if count + ceph_nodes_count > settings.NODES_COUNT - 1:
logger.info("There are not enough nodes to redeploy with "
"ceph nodes pool size. Go to the next config")
continue
self.add_ceph_nodes(count, ceph_nodes_count)
self.show_step(10, details=conf['name'])
if not self.update_cluster_attributes(new_attrs):
fail_trigger = True
failed_confs.append(conf['name'])
logger.error(
"Failed to update cluster attributes with changed one")
continue
self.show_step(11, details=conf['name'])
if not self.deploy_cluster():
logger.error("Redeployment of cluster {0} "
"with config name {1} was failed. "
"Go to the next config".format(
self.cluster_name, conf['name']))
fail_trigger = True
failed_confs.append(conf['name'])
continue
# Run ostf
self.show_step(12, details=conf['name'])
if not self.run_ostf():
fail_trigger = True
failed_confs.append(conf['name'])
logger.error("Failed to pass OSTF tests for redeployed "
"cluster with config {}".format(conf['name']))
continue
logger.info("Redeployment and OSTF were successfully "
"executed for cluster {}".format(self.cluster_name))
self.show_step(13, details=conf['name'])
if fail_trigger:
assert_false(
fail_trigger, "A few configurations were failed: {} "
"Please, check logs".format(failed_confs))
def add_custom_nodegroup(self):
"""Add new nodegroup to operational environment
Scenario:
1. Revert snapshot with operational cluster
2. Create new nodegroup for the environment and configure
it's networks
3. Bootstrap slave node from custom-2 nodegroup
4. Add node from new nodegroup to the environment with compute role
5. Run network verification
6. Deploy changes
7. Run network verification
8. Run OSTF
9. Check that nodes from 'default' nodegroup can reach nodes
from new nodegroup via management and storage networks
Duration 50m
Snapshot add_custom_nodegroup
"""
self.show_step(1, initialize=True)
self.env.revert_snapshot('deploy_neutron_tun_ha_nodegroups')
cluster_id = self.fuel_web.get_last_created_cluster()
self.fuel_web.assert_nodes_in_ready_state(cluster_id)
asserts.assert_true(
not any(ng['name'] == NODEGROUPS[2]['name']
for ng in self.fuel_web.client.get_nodegroups()),
'Custom nodegroup {0} already '
'exists!'.format(NODEGROUPS[2]['name']))
self.show_step(2)
new_nodegroup = self.fuel_web.client.create_nodegroup(
cluster_id, NODEGROUPS[2]['name'])
logger.debug('Updating custom nodegroup ID in network configuration..')
network_config_new = self.fuel_web.client.get_networks(cluster_id)
asserts.assert_true(self.netconf_all_groups is not None,
'Network configuration for nodegroups is empty!')
for network in self.netconf_all_groups['networks']:
if network['group_id'] is not None and \
not any(network['group_id'] == ng['id']
for ng in self.fuel_web.client.get_nodegroups()):
network['group_id'] = new_nodegroup['id']
for new_network in network_config_new['networks']:
if new_network['name'] == network['name'] and \
new_network['group_id'] == network['group_id']:
network['id'] = new_network['id']
self.fuel_web.client.update_network(
cluster_id, self.netconf_all_groups['networking_parameters'],
self.netconf_all_groups['networks'])
self.show_step(3)
self.env.bootstrap_nodes([self.env.d_env.nodes().slaves[6]])
self.show_step(4)
self.fuel_web.update_nodes(
cluster_id, {'slave-07': [['compute'], new_nodegroup['name']]},
True, False)
self.show_step(5)
self.fuel_web.verify_network(cluster_id)
self.show_step(6)
self.fuel_web.deploy_cluster_wait(cluster_id)
self.show_step(7)
self.fuel_web.verify_network(cluster_id)
self.show_step(8)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.show_step(9)
primary_ctrl = self.fuel_web.get_nailgun_node_by_devops_node(
self.fuel_web.get_nailgun_primary_node(
slave=self.env.d_env.nodes().slaves[0]))
with self.fuel_web.get_ssh_for_node('slave-07') as remote:
new_node_networks = utils.get_net_settings(remote)
for interface in ('br-storage', 'br-mgmt'):
if interface in new_node_networks:
logger.info("Checking new node is accessible from primary "
"controller via {0} interface.".format(interface))
for ip in new_node_networks[interface]['ip_addresses']:
address = ip.split('/')[0]
result = check_ping(primary_ctrl['ip'], address, timeout=3)
asserts.assert_true(
result, "New node isn't accessible from "
"primary controller via {0} interface"
": {1}.".format(interface, result))
self.env.make_snapshot("add_custom_nodegroup")
def delete_cluster_with_custom_nodegroup(self):
"""Delete env, check nodes from custom nodegroup can't bootstrap
Scenario:
1. Revert snapshot with cluster with nodes in custom nodegroup
2. Delete cluster
3. Check nodes from custom nodegroup can't bootstrap
4. Reset nodes from custom nodegroup
5. Check nodes from custom nodegroup can't bootstrap
Duration 15m
"""
self.show_step(1, initialize=True)
self.env.revert_snapshot('deploy_controllers_from_custom_nodegroup')
cluster_id = self.fuel_web.get_last_created_cluster()
self.fuel_web.assert_nodes_in_ready_state(cluster_id)
self.show_step(2)
custom_nodes = self.env.d_env.nodes().slaves[3:6]
self.fuel_web.delete_env_wait(cluster_id)
self.show_step(3)
logger.info('Wait five nodes online for 900 seconds..')
wait(lambda: len(self.fuel_web.client.list_nodes()) == 5,
timeout=15 * 60,
timeout_msg='Timeout while waiting five nodes '
'to become online')
logger.info('Wait all nodes from custom nodegroup become '
'in error state..')
# check all custom in error state
for slave in custom_nodes:
wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(slave)[
'status'] == 'error',
timeout=15 * 60,
timeout_msg='Node {} not changed state to '
'error'.format(slave.name))
logger.info('Node {} changed state to error'.format(slave.name))
self.show_step(4)
logger.info('Rebooting nodes from custom nodegroup..')
self.fuel_web.cold_restart_nodes(custom_nodes, wait_online=False)
self.show_step(5)
logger.info('Wait custom nodes are not online for 600 seconds..')
try:
wait(lambda: any(
self.fuel_web.get_nailgun_node_by_devops_node(slave)['online']
for slave in custom_nodes),
timeout=10 * 60)
raise AssertionError('Some nodes online')
except TimeoutError:
logger.info('Nodes are offline')
self.env.make_snapshot("delete_cluster_with_custom_nodegroup")
def basic_env_for_numa_cpu_pinning(self):
"""Basic environment for NUMA CPU pinning
Scenario:
1. Create cluster
2. Add 2 nodes with compute role
3. Add 3 nodes with controller role
4. Verify that quantity of NUMA is equal on node and in Fuel
Snapshot: basic_env_for_numa_cpu_pinning
"""
snapshot_name = 'basic_env_for_numa_cpu_pinning'
self.check_run(snapshot_name)
self.env.revert_snapshot("ready_with_5_slaves")
# TODO(kdemina) Use commomn function for variables asserts
if not settings.KVM_USE:
raise exceptions.FuelQAVariableNotSet(
'KVM_USE', 'true')
if int(settings.HARDWARE['slave_node_cpu']) < 6:
raise exceptions.FuelQAVariableNotSet(
'SLAVE_NODE_CPU', 6)
if int(settings.HARDWARE['numa_nodes']) < 2:
raise exceptions.FuelQAVariableNotSet(
'NUMA_NODES', 2)
if not settings.INTERFACES_DICT['eth0'] == 'ens3':
raise exceptions.FuelQAVariableNotSet(
'IFACE_0', 'ens3')
if not settings.INTERFACES_DICT['eth1'] == 'ens4':
raise exceptions.FuelQAVariableNotSet(
'IFACE_1', 'ens4')
if not settings.INTERFACES_DICT['eth2'] == 'ens5':
raise exceptions.FuelQAVariableNotSet(
'IFACE_2', 'ens5')
elif not settings.INTERFACES_DICT['eth3'] == 'ens6':
raise exceptions.FuelQAVariableNotSet(
'IFACE_3', 'ens6')
elif not settings.INTERFACES_DICT['eth4'] == 'ens7':
raise exceptions.FuelQAVariableNotSet(
'IFACE_4', 'ens7')
elif not settings.INTERFACES_DICT['eth5'] == 'ens8':
raise exceptions.FuelQAVariableNotSet(
'IFACE_5', 'ens8')
elif not settings.ACPI_ENABLE:
raise exceptions.FuelQAVariableNotSet(
'DRIVER_ENABLE_ACPI', 'true')
self.show_step(1)
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings={
"net_provider": 'neutron',
"net_segment_type": settings.NEUTRON_SEGMENT_TYPE
}
)
self.show_step(2)
self.show_step(3)
self.fuel_web.update_nodes(
cluster_id,
{
'slave-01': ['compute'],
'slave-02': ['compute'],
'slave-03': ['controller'],
'slave-04': ['controller'],
'slave-05': ['controller']
})
self.show_step(4)
for node in ('slave-01', 'slave-02'):
target_node = self.fuel_web.get_nailgun_node_by_name(node)
numas_from_fuel = len(
target_node['meta']['numa_topology']['numa_nodes'])
numas_on_remote = utils.get_quantity_of_numa(target_node['ip'])
if not numas_on_remote:
# Fuel handle topology without NUMA as 1 NUMA node
asserts.assert_equal(numas_from_fuel, 1,
"No NUMA nodes on {0} "
"while Fuel shows it "
"has {1}".format(
target_node['ip'], numas_from_fuel))
raise AssertionError("No NUMA nodes on {0}".format(
target_node['ip']))
else:
asserts.assert_equal(numas_on_remote, numas_from_fuel,
"{0} NUMA nodes on {1} "
"while Fuel shows it "
"has {2}".format(
numas_on_remote, target_node['ip'],
numas_from_fuel))
logger.info("There is {0} NUMA nodes on node {1}".format(
numas_on_remote, target_node['ip']))
self.env.make_snapshot(snapshot_name, is_make=True)
def deploy_ceph_ha_nodegroups(self):
"""Deploy HA environment with Neutron VXLAN, Ceph and 2 nodegroups
Scenario:
1. Revert snapshot with ready master node
2. Create cluster with Neutron VXLAN, Ceph and custom nodegroup
3. Exclude 10 first IPs from range for default admin/pxe network
4. Bootstrap slave nodes from both default and custom nodegroups
5. Check that excluded IPs aren't allocated to discovered nodes
6. Add 3 controller + ceph nodes from default nodegroup
7. Add 2 compute + ceph nodes from custom nodegroup
8. Deploy cluster
9. Run network verification
10. Run health checks (OSTF)
11. Check that excluded IPs aren't allocated to deployed nodes
12. Check Ceph health
Duration 110m
Snapshot deploy_ceph_ha_nodegroups
"""
if not MULTIPLE_NETWORKS:
raise SkipTest('MULTIPLE_NETWORKS not enabled')
self.show_step(1, initialize=True)
self.env.revert_snapshot("ready")
self.show_step(2)
cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__,
mode=DEPLOYMENT_MODE_HA,
settings={
'volumes_ceph':
True,
'images_ceph':
True,
'ephemeral_ceph':
True,
'volumes_lvm':
False,
"net_provider":
'neutron',
"net_segment_type":
NEUTRON_SEGMENT['tun'],
'tenant':
'haVxlanCeph',
'user':
'******',
'password':
'******'
})
self.show_step(3)
networks = self.fuel_web.client.get_networks(cluster_id)["networks"]
new_admin_range = self.change_default_admin_range(
networks, number_excluded_ips=10)
wait(lambda: not self.is_update_dnsmasq_running(self.fuel_web.client.
get_tasks()),
timeout=60,
timeout_msg="Timeout exceeded while waiting for task "
"'update_dnsmasq' is finished!")
self.fuel_web.client.update_network(cluster_id, networks=networks)
logger.info("New addresses range for default admin network:"
" {0}".format(new_admin_range))
self.show_step(4)
self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[0:5])
self.show_step(5)
default_ng_nodes = [
self.fuel_web.get_nailgun_node_by_devops_node(node)
for node in self.env.d_env.nodes().slaves[0:3]
]
for node in default_ng_nodes:
asserts.assert_true(
self.is_ip_in_range(node['ip'], *new_admin_range),
"Node '{0}' has IP address '{1}' which "
"is not from defined IP addresses range:"
" {2}!".format(node['fqdn'], node['ip'], new_admin_range))
self.show_step(6)
self.show_step(7)
nodegroup_default = NODEGROUPS[0]['name']
nodegroup_custom = NODEGROUPS[1]['name']
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': [['controller', 'ceph-osd'], nodegroup_default],
'slave-02': [['controller', 'ceph-osd'], nodegroup_default],
'slave-03': [['controller', 'ceph-osd'], nodegroup_default],
'slave-04': [['compute', 'ceph-osd'], nodegroup_custom],
'slave-05': [['compute', 'ceph-osd'], nodegroup_custom],
})
self.show_step(8)
self.fuel_web.deploy_cluster_wait(cluster_id, timeout=150 * 60)
self.show_step(9)
self.fuel_web.verify_network(cluster_id)
self.show_step(10)
self.fuel_web.run_ostf(cluster_id=cluster_id)
self.show_step(11)
group_id = self.fuel_web.get_nodegroup(cluster_id,
name=nodegroup_default)['id']
default_ng_nodes = [
node
for node in self.fuel_web.client.list_cluster_nodes(cluster_id)
if node['group_id'] == group_id
]
for node in default_ng_nodes:
asserts.assert_true(
self.is_ip_in_range(node['ip'], *new_admin_range),
"Node '{0}' has IP address '{1}' which "
"is not from defined IP addresses range:"
" {2}!".format(node['fqdn'], node['ip'], new_admin_range))
self.show_step(12)
self.fuel_web.check_ceph_status(cluster_id)
self.env.make_snapshot("deploy_ceph_ha_nodegroups")
def contrail_plugin_add_delete_controller_node(self):
"""Verify that Controller node can be
deleted and added after deploying
Scenario:
1. Revert snapshot "ready_with_9_slaves"
2. Create cluster
3. Add 3 nodes with Operating system role,
2 nodes with controller role and 1 node with compute role
4. Enable Contrail plugin
5. Deploy cluster with plugin
6. Remove 1 node with controller role.
7. Deploy cluster
8. Add 1 nodes with controller role
9. Deploy cluster
10. Run OSTF tests
Duration 140 min
"""
self._prepare_contrail_plugin(slaves=9)
# create cluster: 3 nodes with Operating system role
# and 1 node with controller role
self.fuel_web.update_nodes(
self.cluster_id,
{
'slave-01': ['base-os'],
'slave-02': ['base-os'],
'slave-03': ['base-os'],
'slave-04': ['controller'],
'slave-05': ['controller'],
'slave-06': ['controller'],
'slave-07': ['compute']
},
custom_names={
'slave-01': 'contrail-1',
'slave-02': 'contrail-2',
'slave-03': 'contrail-3'
}
)
# configure disks on base-os nodes
self.change_disk_size()
# enable plugin in contrail settings
self._activate_plugin()
self.fuel_web.deploy_cluster_wait(self.cluster_id,
check_services=False,
timeout=240 * 60)
# remove one node with controller role
self.fuel_web.update_nodes(
self.cluster_id, {'slave-05': ['controller']}, False, True)
self.fuel_web.deploy_cluster_wait(self.cluster_id,
check_services=False,
timeout=240 * 60)
# add 1 node with controller role and redeploy cluster
self.fuel_web.update_nodes(
self.cluster_id, {'slave-08': ['controller']})
self.fuel_web.deploy_cluster_wait(self.cluster_id,
check_services=False,
timeout=240 * 60)
# TODO
# Tests using north-south connectivity are expected to fail because
# they require additional gateway nodes, and specific contrail
# settings. This mark is a workaround until it's verified
# and tested manually.
# Also workaround according to bug 1457515
# When it will be done 'should_fail=3' and
# 'failed_test_name' parameter should be removed.
# create net and subnet to pass ostf
self._create_net_subnet(self.cluster_id)
self.fuel_web.run_ostf(
cluster_id=self.cluster_id,
should_fail=2,
failed_test_name=[('Check network connectivity '
'from instance via floating IP'),
'Launch instance with file injection']
)
logger.info(self._ostf_msg)
def main():
parser = argparse.ArgumentParser(description="""
Example: python helpers/conf_tempest.py -c 1 \
-n 10.108.10.2 \
-t /home/fuel/tempest/etc/tempest.conf
""")
parser.add_argument("-n",
"--nailgun",
help="Provide nailgun node ip.",
required=True)
parser.add_argument("-c",
"--cluster",
help="Provide cluster id",
required=True)
parser.add_argument("-t",
"--tempest_config",
help="Path where tempest will look for config",
default='/etc/tempest/tempest.conf')
args = parser.parse_args()
conf = TempestConfigState(args.nailgun,
args.cluster,
tempest_conf=args.tempest_config)
conf.configure()
conf.copy_config()
if __name__ == '__main__':
logger.info('Starting tempest config generation.')
main()
logger.info('Finished tempest config generation.')
def ceph_ha_restart(self):
"""Destructive ceph test in HA mode
Scenario:
1. Revert from ceph_ha
2. Waiting up galera and cinder
3. Check ceph status
4. Run OSTF
5. Destroy and remove osd-node
6. Check ceph status
7. Run OSTF
8. Destroy and remove one compute node
9. Check ceph status
10. Run OSTF
11. Cold restart
12. Waiting up galera and cinder
13. Run single OSTF - Create volume and attach it to instance
14. Run OSTF
Duration 30m
Snapshot ceph_ha_restart
"""
self.env.revert_snapshot("ceph_ha")
# Wait until MySQL Galera is UP on some controller
self.fuel_web.wait_mysql_galera_is_up(['slave-01'])
# Wait until Cinder services UP on a controller
self.fuel_web.wait_cinder_is_up(['slave-01'])
cluster_id = self.fuel_web.get_last_created_cluster()
self.fuel_web.check_ceph_status(cluster_id)
# Run ostf
self.fuel_web.run_ostf(cluster_id=cluster_id)
# Destroy and remove osd-node
logger.info("Destroy and remove slave-06")
with self.fuel_web.get_ssh_for_node('slave-06') as remote_ceph:
self.fuel_web.prepare_ceph_to_delete(remote_ceph)
slave_06 = self.env.d_env.get_node(name='slave-06')
nailgun_node_id = self.fuel_web.get_nailgun_node_by_devops_node(
slave_06)['id']
slave_06.destroy()
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
slave_06)['online'],
timeout=30 * 8)
self.fuel_web.delete_node(nailgun_node_id)
self.fuel_web.check_ceph_status(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['sanity', 'smoke', 'ha'],
should_fail=1)
# Destroy and remove compute node
logger.info("Destroy and remove slave-05")
with self.fuel_web.get_ssh_for_node('slave-05') as remote_ceph:
self.fuel_web.prepare_ceph_to_delete(remote_ceph)
slave_05 = self.env.d_env.get_node(name='slave-05')
nailgun_node_id = self.fuel_web.get_nailgun_node_by_devops_node(
slave_05)['id']
slave_05.destroy()
wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node(
slave_05)['online'],
timeout=30 * 8)
self.fuel_web.delete_node(nailgun_node_id)
self.fuel_web.check_ceph_status(cluster_id)
self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1)
# Cold restart
self.fuel_web.cold_restart_nodes(
self.env.d_env.get_nodes(
name__in=['slave-01', 'slave-02', 'slave-03', 'slave-04']))
# Wait for HA services ready
self.fuel_web.assert_ha_services_ready(cluster_id)
# Wait until OpenStack services are UP, should fail 2 services
# because slave-05 (compute+ceph-osd) destroyed. We ignore
# expect fail a test - 'Check openstack services are running'
self.fuel_web.assert_os_services_ready(cluster_id, should_fail=1)
self.fuel_web.check_ceph_status(cluster_id)
# Wait until MySQL Galera is UP on some controller
self.fuel_web.wait_mysql_galera_is_up(['slave-01'])
# Wait until Cinder services UP on a controller
self.fuel_web.wait_cinder_is_up(['slave-01'])
try:
self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['smoke'],
test_name=map_ostf.OSTF_TEST_MAPPING.get(
'Create volume and attach it to instance'))
except AssertionError:
logger.debug("Test failed from first probe,"
" we sleep 60 second try one more time "
"and if it fails again - test will fails ")
time.sleep(180)
self.fuel_web.run_single_ostf_test(
cluster_id,
test_sets=['smoke'],
test_name=map_ostf.OSTF_TEST_MAPPING.get(
'Create volume and attach it to instance'))
self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1)
self.env.make_snapshot("ceph_ha_restart")
def _install_packages(self, remote):
command = "cd " + self._pack_copy_path + " && ./install.sh"
logger.info('The command is %s', command)
remote.execute_async(command)
time.sleep(50)
os.path.isfile(self._add_ub_packag or self._add_cen_packeg)
def deploy_ha_cgroup(self):
"""Deploy cluster in HA mode with enabled cgroups
Scenario:
1. Create cluster
2. Add 3 nodes with controller role
3. Add 1 node with compute role
4. Add 1 node with cinder role
5. Deploy the cluster
6. Check ceph status
7. Run OSTF
Duration 90m
Snapshot deploy_ha_cgroup
"""
self.check_run("deploy_ha_cgroup")
self.env.revert_snapshot("ready_with_5_slaves")
data = {
'tenant': 'cgroup',
'user': '******',
'password': '******',
'net_provider': 'neutron',
'net_segment_type': settings.NEUTRON_SEGMENT['vlan']
}
cluster_id = self.fuel_web.create_cluster(
name=self.__class__.__name__,
mode=settings.DEPLOYMENT_MODE,
settings=data)
self.fuel_web.update_nodes(
cluster_id, {
'slave-01': ['controller'],
'slave-02': ['controller'],
'slave-03': ['controller'],
'slave-04': ['compute'],
'slave-05': ['cinder']
})
cgroup_data = [{
"process": "keystone",
"controller": "cpu",
"limit": "cpu.shares",
"value": 70,
}]
self.fuel_web.client.update_cluster_attributes(
cluster_id, self.generate_attributes(cgroup_data))
# Cluster deploy
self.fuel_web.deploy_cluster_wait(cluster_id)
# Run ostf
self.fuel_web.run_ostf(cluster_id=cluster_id)
# Check that task cgroup was executed
cmd = 'fgrep "MODULAR: cgroups/cgroups.pp" -q /var/log/puppet.log'
n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['controller'])
for nailgun_node in n_ctrls:
logger.info('Check cgroups task on controller node {0}'.format(
nailgun_node["fqdn"]))
self.ssh_manager.check_call(nailgun_node['ip'], cmd)
self.check_cgroups_on_node(nailgun_node, cgroup_data)
self.env.make_snapshot("deploy_ha_cgroup", is_make=True)
def deploy_ha_contrail_plugin(self):
"""Deploy HA Environment with Contrail Plugin
Scenario:
1. Revert snapshot "ready_with_9_slaves"
2. Create cluster
3. Add 3 nodes with Operating system role and
1 node with controller role
4. Enable Contrail plugin
5. Deploy cluster with plugin
6. Add 1 node with compute role
7. Deploy cluster
8. Run OSTF tests
9. Add 2 nodes with controller role and
1 node with compute + cinder role
10. Deploy cluster
11. Run OSTF tests
Duration 140 min
"""
self._prepare_contrail_plugin(slaves=9)
# create cluster: 3 nodes with Operating system role
# and 1 node with controller role
self.fuel_web.update_nodes(
self.cluster_id,
{
'slave-01': ['base-os'],
'slave-02': ['base-os'],
'slave-03': ['base-os'],
'slave-04': ['controller']
},
custom_names={
'slave-01': 'contrail-1',
'slave-02': 'contrail-2',
'slave-03': 'contrail-3'
}
)
# configure disks on base-os nodes
self.change_disk_size()
# enable plugin in contrail settings
self._activate_plugin()
self.fuel_web.deploy_cluster_wait(self.cluster_id)
# create net and subnet
self._create_net_subnet(self.cluster_id)
# add 1 node with compute role and redeploy cluster
self.fuel_web.update_nodes(
self.cluster_id, {'slave-05': ['compute']},)
self.fuel_web.deploy_cluster_wait(self.cluster_id)
# TODO
# Tests using north-south connectivity are expected to fail because
# they require additional gateway nodes, and specific contrail
# settings. This mark is a workaround until it's verified
# and tested manually.
# When it will be done 'should_fail=2' and
# 'failed_test_name' parameter should be removed.
self.fuel_web.run_ostf(
cluster_id=self.cluster_id,
should_fail=2,
failed_test_name=[('Check network connectivity '
'from instance via floating IP'),
'Launch instance with file injection']
)
logger.info(self._ostf_msg)
# add to cluster 2 nodes with controller role and one
# with compute, cinder role and deploy cluster
self.fuel_web.update_nodes(
self.cluster_id,
{
'slave-06': ['controller'],
'slave-07': ['controller'],
'slave-08': ['compute', 'cinder'],
}
)
logger.info(self._ostf_msg)
self.fuel_web.deploy_cluster_wait(self.cluster_id)
# TODO:
# Tests using north-south connectivity are expected to fail because
# they require additional gateway nodes, and specific contrail
# settings. This mark is a workaround until it's verified
# and tested manually.
# When it will be done 'should_fail=2' and
# 'failed_test_name' parameter should be removed.
self.fuel_web.run_ostf(
cluster_id=self.cluster_id,
should_fail=2,
failed_test_name=[('Check network connectivity '
'from instance via floating IP'),
'Launch instance with file injection']
)
def cli_selected_nodes_deploy(self):
"""Create and deploy environment using Fuel CLI and check CN name
is equal to the public name passed via UI (user-owned cert)
Scenario:
1. Create environment using fuel-qa
2. Create a cluster using Fuel CLI
3. Add floating ranges for public network
4. Get cluster settings
5. Provision a controller node using Fuel CLI
6. Provision two compute+cinder nodes using Fuel CLI
7. Deploy the controller node using Fuel CLI
8. Deploy the compute+cinder nodes using Fuel CLI
9. Compare network settings after compute deployment task
10. Verify network
11. Check that all services work by 'https'
12. Check that all services have domain name
13. Find 'CN' value at the output:
CN value is equal to the value specified
at certificate provided via Fuel UI
14. Find keypair data at the output:
Keypair data is equal to the value specified
at certificate provided via Fuel UI
15. Compare floating ranges
16. Get deployment-info
17. Get cluster settings after deployment task
18. Compare cluster settings after deploy and before deploy
19. Run OSTF
Duration 50m
"""
self.env.revert_snapshot("ready_with_3_slaves")
node_ids = sorted(
[node['id'] for node in self.fuel_web.client.list_nodes()])
release_id = self.fuel_web.get_releases_list_for_os(
release_name=OPENSTACK_RELEASE)[0]
admin_ip = self.ssh_manager.admin_ip
# Create an environment
self.show_step(1)
if NEUTRON_SEGMENT_TYPE:
nst = '--nst={0}'.format(NEUTRON_SEGMENT_TYPE)
else:
nst = ''
self.show_step(2)
cmd = ('fuel env create --name={0} --release={1} {2} --json'.format(
self.__class__.__name__, release_id, nst))
env_result =\
self.ssh_manager.execute_on_remote(admin_ip, cmd,
jsonify=True)['stdout_json']
cluster_id = env_result['id']
self.show_step(3)
# Update network parameters
self.update_cli_network_configuration(cluster_id)
# Change floating ranges
current_floating_range = self.get_floating_ranges(cluster_id)
logger.info(
"Current floating ranges: {0}".format(current_floating_range))
first_floating_address = current_floating_range[0][0]
logger.info(
"First floating address: {0}".format(first_floating_address))
last_floating_address = current_floating_range[0][1]
logger.info("Last floating address: {0}".format(last_floating_address))
new_floating_range = generate_floating_ranges(first_floating_address,
last_floating_address,
10)
logger.info("New floating range: {0}".format(new_floating_range))
self.change_floating_ranges(cluster_id, new_floating_range)
# Update SSL configuration
self.update_ssl_configuration(cluster_id)
# Get cluster settings before deploy
self.show_step(4)
cluster_settings = self.download_settings(cluster_id)
self.show_step(5)
# Add and provision a controller node
logger.info("Add to the cluster \
and start provisioning a controller node [{0}]".format(node_ids[0]))
cmd = ('fuel --env-id={0} node set --node {1}\
--role=controller'.format(cluster_id, node_ids[0]))
self.ssh_manager.execute_on_remote(admin_ip, cmd)
self.update_node_interfaces(node_ids[0])
cmd = ('fuel --env-id={0} node --provision --node={1} --json'.format(
cluster_id, node_ids[0]))
task = self.ssh_manager.execute_on_remote(admin_ip, cmd,
jsonify=True)['stdout_json']
self.assert_cli_task_success(task, timeout=30 * 60)
self.show_step(6)
# Add and provision 2 compute+cinder
logger.info("Add to the cluster and start provisioning two "
"compute+cinder nodes [{0},{1}]".format(
node_ids[1], node_ids[2]))
cmd = ('fuel --env-id={0} node set --node {1},{2} \
--role=compute,cinder'.format(cluster_id, node_ids[1], node_ids[2]))
self.ssh_manager.execute_on_remote(admin_ip, cmd)
for node_id in (node_ids[1], node_ids[2]):
self.update_node_interfaces(node_id)
cmd = ('fuel --env-id={0} node --provision \
--node={1},{2} --json'.format(cluster_id, node_ids[1], node_ids[2]))
task = self.ssh_manager.execute_on_remote(admin_ip, cmd,
jsonify=True)['stdout_json']
self.assert_cli_task_success(task, timeout=10 * 60)
self.show_step(7)
# Deploy the controller node
cmd = ('fuel --env-id={0} node --deploy --node {1} --json'.format(
cluster_id, node_ids[0]))
task = self.ssh_manager.execute_on_remote(admin_ip, cmd,
jsonify=True)['stdout_json']
self.assert_cli_task_success(task, timeout=60 * 60)
self.assert_all_tasks_completed(cluster_id=cluster_id)
self.show_step(8)
# Deploy the compute nodes
cmd = ('fuel --env-id={0} node --deploy --node {1},{2} --json'.format(
cluster_id, node_ids[1], node_ids[2]))
task = self.ssh_manager.execute_on_remote(admin_ip, cmd,
jsonify=True)['stdout_json']
self.wait_cli_task_status(task=task, status='running')
# Fuel 9.1 is async, so we should wait for real task start
network_settings = self.get_networks(cluster_id)
self.assert_cli_task_success(task, timeout=30 * 60)
self.assert_all_tasks_completed(cluster_id=cluster_id)
# Verify networks
self.show_step(9)
network_configuration = self.get_net_config_cli()
assert_equal(network_settings,
network_configuration,
message='Network settings are not equal before'
' and after deploy')
self.show_step(10)
self.fuel_web.verify_network(cluster_id)
controller_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['controller'])
# Get controller ip address
controller_node = controller_nodes[0]['ip']
# Get endpoint list
endpoint_list = self.get_endpoints(controller_node)
logger.info(endpoint_list)
# Check protocol and domain names for endpoints
self.show_step(11)
self.show_step(12)
for endpoint in endpoint_list:
logger.debug(("Endpoint {0} use protocol {1}\
and have domain name {2}".format(endpoint['service_name'],
endpoint['protocol'],
endpoint['domain'])))
assert_equal(endpoint['protocol'],
"https",
message=("Endpoint {0} don't use https.".format(
endpoint['service_name'])))
assert_equal(endpoint['domain'],
SSL_CN,
message=("{0} domain name not equal {1}.".format(
endpoint['service_name'], SSL_CN)))
self.show_step(13)
current_ssl_cn = self.get_current_ssl_cn(controller_node)
logger.info(("CN before cluster deploy {0} \
and after deploy {1}".format(SSL_CN, current_ssl_cn)))
assert_equal(SSL_CN, current_ssl_cn, message="SSL CNs are not equal")
self.show_step(14)
with open(PATH_TO_PEM) as pem_file:
old_ssl_keypair = pem_file.read().strip()
current_ssl_keypair = self.get_current_ssl_keypair(controller_node)
logger.info("SSL keypair before cluster deploy:\n"
"{0}\n"
"and after deploy:\n"
"{1}".format(old_ssl_keypair, current_ssl_keypair))
assert_equal(old_ssl_keypair,
current_ssl_keypair,
message="SSL keypairs are not equal")
self.show_step(15)
actual_floating_ranges = self.hiera_floating_ranges(controller_node)
logger.info(
"Current floating ranges: {0}".format(actual_floating_ranges))
assert_equal(actual_floating_ranges,
new_floating_range,
message="Floating ranges are not equal")
# Get deployment task id
task_id = self.get_first_task_id_by_name(cluster_id, 'deployment')
self.show_step(16)
# Get deployment info
self.get_deployment_info_cli(task_id)
self.show_step(17)
# Get cluster settings after deploy
cluster_config = self.get_cluster_config_cli(task_id)
self.show_step(18)
# Compare cluster settings
assert_equal(cluster_settings,
cluster_config,
message='Cluster settings are not equal before'
' and after deploy')
self.show_step(19)
# Run OSTF
self.fuel_web.run_ostf(cluster_id=cluster_id,
test_sets=['ha', 'smoke', 'sanity'])
self.env.make_snapshot("cli_selected_nodes_deploy", is_make=True)
def apply_relative_cgroups_after_deploy(self):
"""Apply relative cgroups limits to services
Scenario:
1. Revert snapshot deploy_ha_cgroup
2. Configure and validate cgroups for mysqld, rabbitmq
and keystone with relative memory count
Duration 15m
"""
self.show_step(1)
self.env.revert_snapshot("deploy_ha_cgroup")
cluster_id = self.fuel_web.get_last_created_cluster()
n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles(
cluster_id, ['controller'])
ctrl_ids = ",".join(
[str(nailgun_node['id']) for nailgun_node in n_ctrls])
self.show_step(2)
cgroups = [
{
"process": "mysqld",
"controller": "memory",
"limit": "memory.swappiness",
"value": 0
},
{
"process": "mysqld",
"controller": "memory",
"limit": "memory.soft_limit_in_bytes",
"value": "%5,10,3000"
},
{
"process": "rabbitmq",
"controller": "blkio",
"limit": "blkio.weight",
"value": 500
},
{
"process": "rabbitmq",
"controller": "memory",
"limit": "memory.soft_limit_in_bytes",
"value": "%99,10,250"
},
{
"process": "keystone",
"controller": "cpu",
"limit": "cpu.shares",
"value": 50
},
{
"process": "keystone",
"controller": "memory",
"limit": "memory.soft_limit_in_bytes",
"value": "%1,250,2500"
},
]
self.apply_cgroups(cgroups, ctrl_ids)
memory = float("".join(
self.ssh_manager.execute(n_ctrls[0]["ip"],
"facter memorysize_mb")["stdout"]))
for cgroup in cgroups:
if cgroup["limit"] == "memory.soft_limit_in_bytes":
# pylint: disable=no-member
percent, min_mem, max_mem = cgroup["value"].split(",")
# pylint: enable=no-member
percent = int(percent.replace("%", "")) * memory / 100
min_mem, max_mem = int(min_mem), int(max_mem)
value = sorted((min_mem, percent, max_mem))[1]
cgroup["value"] = int(value * 1024 * 1024)
logger.info("New cgroups to verify: {}".format(cgroups))
for nailgun_node in n_ctrls:
self.check_cgroups_on_node(nailgun_node, cgroups)
def replace_fuel_nailgun_rpm(environment):
"""
Replace fuel_nailgun*.rpm from review
environment - Environment Model object - self.env
"""
logger.info("Patching fuel-nailgun")
if not settings.UPDATE_FUEL:
raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True')
pack_path = '/var/www/nailgun/fuel-nailgun/'
full_pack_path = os.path.join(pack_path, 'fuel-nailgun*.noarch.rpm')
logger.info('Package path {0}'.format(full_pack_path))
with environment.d_env.get_admin_remote() as remote:
remote.upload(settings.UPDATE_FUEL_PATH.rstrip('/'), pack_path)
# Check old fuel-nailgun package
cmd = "rpm -q fuel-nailgun"
old_package = environment.base_actions.execute(cmd, exit_code=0)
logger.info('Current package version of '
'fuel-nailgun: {0}'.format(old_package))
cmd = "rpm -qp {0}".format(full_pack_path)
new_package = environment.base_actions.execute(cmd)
logger.info("Updating package {0} with {1}".format(old_package,
new_package))
if old_package == new_package:
logger.debug('Looks like package from review '
'was installed during setups of master node')
return
# stop services
service_list = ['assassind', 'receiverd', 'nailgun', 'statsenderd']
[
environment.base_actions.execute('systemctl stop {0}'.format(service),
exit_code=0)
for service in service_list
]
# stop statistic services
[
environment.base_actions.execute('systemctl stop {0}'.format(service),
exit_code=0)
for service in get_oswl_services_names(environment)
]
# Drop nailgun db manage.py dropdb
cmd = 'manage.py dropdb'
environment.base_actions.execute(cmd, exit_code=0)
# Delete package
logger.info("Delete package {0}".format(old_package))
cmd = "rpm -e fuel-nailgun"
environment.base_actions.execute(cmd, exit_code=0)
logger.info("Install package {0}".format(new_package))
cmd = "rpm -Uvh --oldpackage {0}".format(full_pack_path)
environment.base_actions.execute(cmd, exit_code=0)
cmd = "rpm -q fuel-nailgun"
installed_package = environment.base_actions.execute(cmd, exit_code=0)
assert_equal(installed_package, new_package,
"The new package {0} was not installed".format(new_package))
cmd = ('puppet apply --debug '
'/etc/puppet/modules/fuel/examples/nailgun.pp')
environment.base_actions.execute(cmd, exit_code=0)
with environment.d_env.get_admin_remote() as remote:
res = remote.execute(
"fuel release --sync-deployment-tasks --dir /etc/puppet/")
assert_equal(res['exit_code'], 0,
'Failed to sync tasks with result {0}'.format(res))
def get_defaults():
"""Get default parameters from config.yaml."""
with open('plugin_test/helpers/config.yaml') as config:
defaults = yaml.load(config.read())
logger.info(''.format(defaults))
return defaults
