def ceph_multinode_with_cinder(self): """Deploy ceph with cinder in simple mode Scenario: 1. Create cluster 2. Add 1 node with controller role 3. Add 1 node with compute role 4. Add 2 nodes with cinder and ceph OSD roles 5. Deploy the cluster 6. Check ceph status 7. Check partitions on controller node Snapshot ceph_multinode_with_cinder """ if settings.OPENSTACK_RELEASE == settings.OPENSTACK_RELEASE_REDHAT: raise SkipTest() self.env.revert_snapshot("ready") self.env.bootstrap_nodes(self.env.nodes().slaves[:4]) cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=settings.DEPLOYMENT_MODE_SIMPLE, settings={ 'volumes_ceph': False, 'images_ceph': True, 'volumes_lvm': True } ) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller'], 'slave-02': ['compute'], 'slave-03': ['cinder', 'ceph-osd'], 'slave-04': ['cinder', 'ceph-osd'] } ) # Cluster deploy self.fuel_web.deploy_cluster_wait(cluster_id) check_ceph_health(self.env.get_ssh_to_remote_by_name('slave-01')) disks = self.fuel_web.client.get_node_disks( self.fuel_web.get_nailgun_node_by_name('slave-01')['id']) logger.info("Current disk partitions are: \n{d}".format(d=disks)) logger.info("Check unallocated space") # We expect failure here only for release 5.0 due to bug # https://bugs.launchpad.net/fuel/+bug/1306625, so it is # necessary to assert_true in the next release. assert_false( checkers.check_unallocated_space(disks, contr_img_ceph=True), "Check unallocated space on controller") # Run ostf self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ceph_multinode_with_cinder")
def verify_services(self): """Check that the correct amount of collector processes are running. :returns: list of process IDs indexed by node and process :rtype: dict """ pids = {} processes_count = { "collectd": 1, "collectdmon": 1 } if self.settings.version.startswith("0.9"): processes_count["hekad"] = 1 else: # Starting with 0.10, there are one collector for logs and one for # metrics processes_count["hekad"] = 2 online_nodes = [node for node in self.helpers.get_all_ready_nodes() if node["online"]] for node in online_nodes: pids[node["name"]] = {} with self.env.d_env.get_ssh_to_remote(node["ip"]) as remote: for process, count in processes_count.items(): logger.info("Checking process {0} on node {1}".format( process, node["name"] )) pids[node["name"]][process] = ( self.checkers.check_process_count( remote, process, count)) return pids
def setup_environment(self, custom=settings.CUSTOM_ENV, build_images=settings.BUILD_IMAGES, iso_connect_as=settings.ADMIN_BOOT_DEVICE, security=settings.SECURITY_TEST): # Create environment and start the Fuel master node admin = self.d_env.nodes().admin self.d_env.start([admin]) logger.info("Waiting for admin node to start up") wait(lambda: admin.driver.node_active(admin), 60) logger.info("Proceed with installation") # update network parameters at boot screen admin.send_keys(self.get_keys(admin, custom=custom, build_images=build_images, iso_connect_as=iso_connect_as)) if settings.SHOW_FUELMENU: self.wait_for_fuelmenu() else: self.wait_for_provisioning() self.set_admin_ssh_password() self.wait_for_external_config() if custom: self.setup_customisation() if security: nessus_node = NessusActions(self.d_env) nessus_node.add_nessus_node() # wait while installation complete self.admin_actions.modify_configs(self.d_env.router()) self.kill_wait_for_external_config() self.wait_bootstrap() self.admin_actions.wait_for_fuel_ready()
def upload_packages(self, local_packages_dir, centos_repo_path, ubuntu_repo_path, clean_target=False): logger.info("Upload fuel's packages from directory {0}." .format(local_packages_dir)) centos_files_count = 0 ubuntu_files_count = 0 if centos_repo_path: centos_files_count = self.ssh_manager.cond_upload( ip=self.admin_ip, source=local_packages_dir, target=os.path.join(centos_repo_path, 'Packages'), condition="(?i).*\.rpm$", clean_target=clean_target ) if centos_files_count > 0: regenerate_centos_repo(centos_repo_path) if ubuntu_repo_path: ubuntu_files_count = self.ssh_manager.cond_upload( ip=self.admin_ip, source=local_packages_dir, target=os.path.join(ubuntu_repo_path, 'pool/main'), condition="(?i).*\.deb$", clean_target=clean_target ) if ubuntu_files_count > 0: regenerate_ubuntu_repo(ubuntu_repo_path) return centos_files_count, ubuntu_files_count
def get_oswl_services_names(): cmd = "systemctl list-units| grep oswl_ | awk '{print $1}'" result = SSHManager().execute_on_remote( SSHManager().admin_ip, cmd)['stdout_str'].strip() logger.info('list of statistic services {0}'.format( result.split('\n'))) return result.split('\n')
def build_bootstrap_image(self, **kwargs): simple_fields = \ ("ubuntu-release", "http-proxy", "https-proxy", "script", "label", "extend-kopts", "kernel-flavor", "root-ssh-authorized-file", "output-dir", "image-build-dir") list_fields = ("repo", "direct-repo-addr", "package", "extra-dir") flag_fields = ("activate", ) command = "fuel-bootstrap build " for field in simple_fields: if kwargs.get(field) is not None: command += "--{0} {1} ".format(field, kwargs.get(field)) for field in list_fields: if kwargs.get(field) is not None: for value in kwargs.get(field): command += "--{0} {1} ".format(field, value) for field in flag_fields: if kwargs.get(field) is not None: command += "--{0} ".format(field) logger.info("Building bootstrap image: {0}".format(command)) result = self.ssh_manager.execute_on_remote( ip=self.admin_ip, cmd=command, )['stdout_str'] logger.info("Bootstrap image has been built: {0}".format(result)) uuid = self.parse_uuid(result)[0] path = os.path.join(kwargs.get("output-dir", "/tmp"), "{0}.tar.gz".format(uuid)) return uuid, path
def describe_environment(self): """Environment :rtype : Environment """ environment = self.manager.environment_create(self.env_name) networks = [] interfaces = settings.INTERFACE_ORDER if self.multiple_cluster_networks: logger.info('Multiple cluster networks feature is enabled!') if settings.BONDING: interfaces = settings.BONDING_INTERFACES.keys() for name in interfaces: networks.append(self.create_networks(name, environment)) for name in self.node_roles.admin_names: self.describe_admin_node(name, networks) for name in self.node_roles.other_names: if self.multiple_cluster_networks: networks1 = [net for net in networks if net.name in settings.NODEGROUPS[0]['pools']] networks2 = [net for net in networks if net.name in settings.NODEGROUPS[1]['pools']] # If slave index is even number, then attach to # it virtual networks from the second network group. if int(name[-2:]) % 2 == 1: self.describe_empty_node(name, networks1) elif int(name[-2:]) % 2 == 0: self.describe_empty_node(name, networks2) else: self.describe_empty_node(name, networks) return environment
def modify_configs(self, router): # Slave nodes should use the gateway of 'admin' network as the default # gateway during provisioning and as an additional DNS server. fuel_settings = self.get_fuel_settings() fuel_settings['DNS_UPSTREAM'] = router fuel_settings['ADMIN_NETWORK']['dhcp_gateway'] = router if FUEL_USE_LOCAL_NTPD: # Try to use only ntpd on the host as the time source # for admin node cmd = 'ntpdate -p 4 -t 0.2 -ub {0}'.format(router) if not self.ssh_manager.execute(ip=self.admin_ip, cmd=cmd)['exit_code']: # Local ntpd on the host is alive, so # remove all NTP sources and add the host instead. logger.info("Switching NTPD on the Fuel admin node to use " "{0} as the time source.".format(router)) ntp_keys = [k for k in fuel_settings.keys() if re.match(r'^NTP', k)] for key in ntp_keys: fuel_settings.pop(key) fuel_settings['NTP1'] = router if MIRROR_UBUNTU: fuel_settings['BOOTSTRAP']['repos'] = \ replace_repos.replace_ubuntu_repos( { 'value': fuel_settings['BOOTSTRAP']['repos'] }, upstream_host='archive.ubuntu.com') logger.info("Replace default Ubuntu mirror URL for " "bootstrap image in Fuel settings") self.save_fuel_settings(fuel_settings)
def check_ceilometer_resource_functionality(self): logger.info("Start checking Ceilometer Resource API") fail_msg = "Failed to get resource list." msg = "getting resources list" resources_list = self.helpers.verify( 600, self.ceilometer_client.resources.list, 1, fail_msg, msg, limit=10) for resource in resources_list: # We need to check that resource_id doesn't contain char '/' # because if it is GET resource request fails if "/" not in resource.resource_id: resource_id = resource.resource_id break fail_msg = ("Failed to find '{}' resource with certain resource " "ID.".format(resource_id)) msg = ("searching '{}' resource with certain resource " "ID".format(resource_id)) self.helpers.verify(60, self.ceilometer_client.resources.get, 2, fail_msg, msg, resource_id=resource_id) fail_msg = "Failed to get meters list." msg = "getting meters list" self.helpers.verify(60, self.ceilometer_client.meters.list, 3, fail_msg, msg, limit=10) fail_msg = "Failed to get unique meters list." msg = "getting unique meters list" self.helpers.verify(60, self.ceilometer_client.meters.list, 4, fail_msg, msg, limit=10, unique=True)
def _create_network_resources(self, tenant_id): """This method creates network resources. It creates a network, an internal subnet on the network, a router and links the network to the router. All resources created by this method will be automatically deleted. """ logger.info("Creating network resources...") net_name = "ostf-autoscaling-test-service-net" net_body = { "network": { "name": net_name, "tenant_id": tenant_id } } ext_net = None net = None for network in self.neutron_cli.list_networks()["networks"]: if not net and network["name"] == net_name: net = network if not ext_net and network["router:external"]: ext_net = network if not net: net = self.neutron_cli.create_network(net_body)["network"] subnet = self.helpers.os_conn.create_subnet( "sub" + net_name, net["id"], "10.1.7.0/24", tenant_id=tenant_id ) router_name = 'ostf-autoscaling-test-service-router' router = self.helpers.os_conn.create_router( router_name, self.helpers.os_conn.get_tenant("admin")) self.neutron_cli.add_interface_router( router["id"], {"subnet_id": subnet["id"]}) return net["id"]
def __init__(self, url, keystone_url, credentials, **kwargs): logger.info('Initiate HTTPClient with url %s', url) self.url = url self.keystone_url = keystone_url self.creds = dict(credentials, **kwargs) self.keystone = None self.opener = urllib2.build_opener(urllib2.HTTPHandler)
def post(self, endpoint, data=None, content_type="application/json"): if not data: data = {} logger.info('self url is %s' % self.url) req = urllib2.Request(self.url + endpoint, data=json.dumps(data)) req.add_header('Content-Type', content_type) return self._open(req)
def generate_facts(ip): ssh_manager = SSHManager() facter_dir = '/var/lib/puppet/lib/facter' exluded_facts = ['naily.rb'] if not ssh_manager.isdir_on_remote(ip, facter_dir): ssh_manager.mkdir_on_remote(ip, facter_dir) logger.debug('Directory {0} was created'.format(facter_dir)) ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir)) logger.debug('rb files were removed from {0}'.format(facter_dir)) facts_files = ssh_manager.execute_on_remote( ip, 'find /etc/puppet/modules/ -wholename "*/lib/facter/*.rb"')['stdout'] facts_files = [i.strip() for i in facts_files] logger.debug('The following facts {0} will' ' be copied to {1}'.format(facts_files, facter_dir)) for fact in facts_files: if not fact or re.sub(r'.*/', '', fact) in exluded_facts: continue ssh_manager.execute_on_remote(ip, 'cp {0} {1}/'.format(fact, facter_dir)) logger.debug('Facts were copied') ssh_manager.execute_on_remote(ip, 'facter -p -y > /tmp/facts.yaml') logger.info('Facts yaml was created') ssh_manager.execute_on_remote(ip, 'rm -f {0}/*.rb'.format(facter_dir)) logger.debug('rb files were removed from {0}'.format(facter_dir))
def assert_cli_task_success( self, task, remote, timeout=70 * 60, interval=20): logger.info('Wait {timeout} seconds for task: {task}' .format(timeout=timeout, task=task)) start = time.time() try: wait( lambda: self.get_task( remote, task['id'])['status'] != 'running', interval=interval, timeout=timeout ) except TimeoutError: raise TimeoutError( "Waiting timeout {timeout} sec was reached for task: {task}" .format(task=task["name"], timeout=timeout)) took = time.time() - start task = self.get_task(remote, task['id']) logger.info('Task finished in {took} seconds with the result: {task}' .format(took=took, task=task)) assert_equal( task['status'], 'ready', "Task '{name}' has incorrect status. {} != {}".format( task['status'], 'ready', name=task["name"] ) )
def replace_centos_bootstrap(environment): """Replaced initramfs.img in /var/www/nailgun/ with re-builded with review code environment - Environment Model object - self.env """ logger.info("Updating bootstrap") if not settings.UPDATE_FUEL: raise Exception("{} variable don't exist" .format(settings.UPDATE_FUEL)) try: rebuilded_bootstrap = '/var/initramfs.img.updated' with environment.d_env.get_admin_remote() as remote: checkers.check_file_exists( remote, '{0}'.format(rebuilded_bootstrap)) logger.info("Assigning new bootstrap from {}" .format(rebuilded_bootstrap)) bootstrap = "/var/www/nailgun/bootstrap" cmd = ("mv {0}/initramfs.img /var/initramfs.img;" "cp /var/initramfs.img.updated {0}/initramfs.img;" "chmod +r {0}/initramfs.img;" ).format(bootstrap) result = remote.execute(cmd) assert_equal(result['exit_code'], 0, ('Failed to assign bootstrap {}' ).format(result)) cmd = "cobbler sync" environment.base_actions.execute(cmd, exit_code=0) except Exception as e: logger.error("Could not update bootstrap {e}".format(e=e)) raise
def store_astute_yaml_for_one_node(nailgun_node): ssh_manager = SSHManager() if 'roles' not in nailgun_node: return None errmsg = 'Downloading "{0}.yaml" from the {1} failed' msg = 'File "{0}.yaml" was downloaded from the {1}' nodename = nailgun_node['name'] ip = nailgun_node['ip'] for role in nailgun_node['roles']: filename = '{0}/{1}-{2}-{3}.yaml'.format(settings.LOGS_DIR, func_name, nodename, role) if not ssh_manager.isfile_on_remote(ip, '/etc/{0}.yaml'.format(role)): role = 'primary-' + role if ssh_manager.download_from_remote(ip, '/etc/{0}.yaml'.format(role), filename): logger.info(msg.format(role, nodename)) else: logger.error(errmsg.format(role, nodename)) if settings.DOWNLOAD_FACTS: fact_filename = re.sub(r'-\w*\.', '-facts.', filename) generate_facts(ip) if ssh_manager.download_from_remote(ip, '/tmp/facts.yaml', fact_filename): logger.info(msg.format('facts', nodename)) else: logger.error(errmsg.format('facts', nodename))
def deploy_one_node(self): """Deploy cluster with controller node only Scenario: 1. Create cluster 2. Add 1 node with controller role 3. Deploy the cluster 4. Validate cluster was set up correctly, there are no dead services, there are no errors in logs Duration 20m """ self.env.revert_snapshot("ready") self.fuel_web.client.list_nodes() self.env.bootstrap_nodes( self.env.d_env.nodes().slaves[:1]) cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=DEPLOYMENT_MODE, ) logger.info('Cluster is {!s}'.format(cluster_id)) self.fuel_web.update_nodes( cluster_id, {'slave-01': ['controller']} ) self.fuel_web.deploy_cluster_wait(cluster_id) os_conn = os_actions.OpenStackActions( self.fuel_web.get_public_vip(cluster_id)) self.fuel_web.assert_cluster_ready(os_conn, smiles_count=4) self.fuel_web.run_single_ostf_test( cluster_id=cluster_id, test_sets=['sanity'], test_name=('fuel_health.tests.sanity.test_sanity_identity' '.SanityIdentityTest.test_list_users'))
def wrapper(*args, **kwargs): result = func(*args, **kwargs) try: if settings.UPLOAD_PATCHSET: if not settings.GERRIT_REFSPEC: raise ValueError('REFSPEC should be set for CI tests.') logger.info("Uploading new patchset from {0}" .format(settings.GERRIT_REFSPEC)) remote = SSHClient(args[0].admin_node_ip, username='******', password='******') remote.upload(settings.PATCH_PATH.rstrip('/'), '/tmp/fuel-ostf') remote.execute('source /opt/fuel_plugins/ostf/bin/activate; ' 'cd /tmp/fuel-ostf; python setup.py develop') remote.execute('/etc/init.d/supervisord restart') helpers.wait( lambda: "RUNNING" in remote.execute("supervisorctl status ostf | awk\ '{print $2}'")['stdout'][0], timeout=60) logger.info("OSTF status: RUNNING") except Exception as e: logger.error("Could not upload patch set {e}".format(e=e)) raise return result
def check_mysql(remote, node_name): check_cmd = 'pkill -0 -x mysqld' check_crm_cmd = ('crm resource status clone_p_mysql |' ' grep -q "is running on: $HOSTNAME"') check_galera_cmd = ("mysql --connect_timeout=5 -sse \"SELECT" " VARIABLE_VALUE FROM" " information_schema.GLOBAL_STATUS" " WHERE VARIABLE_NAME" " = 'wsrep_local_state_comment';\"") try: wait(lambda: remote.execute(check_cmd)['exit_code'] == 0, timeout=300) logger.info('MySQL daemon is started on {0}'.format(node_name)) except TimeoutError: logger.error('MySQL daemon is down on {0}'.format(node_name)) raise _wait(lambda: assert_equal(remote.execute(check_crm_cmd)['exit_code'], 0, 'MySQL resource is NOT running on {0}'.format( node_name)), timeout=60) try: wait(lambda: ''.join(remote.execute( check_galera_cmd)['stdout']).rstrip() == 'Synced', timeout=600) except TimeoutError: logger.error('galera status is {0}'.format(''.join(remote.execute( check_galera_cmd)['stdout']).rstrip())) raise
def _cluster_from_config(self, config): """Create cluster from predefined config.""" slaves = len(config.get('nodes')) cluster_name = config.get('name', self._context.__name__) snapshot_name = "ready_cluster_{}".format(cluster_name) if self.check_run(snapshot_name): self.env.revert_snapshot(snapshot_name) cluster_id = self.fuel_web.client.get_cluster_id(cluster_name) self._context._storage['cluster_id'] = cluster_id logger.info("Getted deployed cluster from snapshot") return True elif self.get_ready_slaves(slaves): logger.info("Create env {}".format(cluster_name)) cluster_id = self.fuel_web.create_cluster( name=cluster_name, mode=config.get('mode', settings.DEPLOYMENT_MODE), settings=config.get('settings', {}) ) self._context._storage['cluster_id'] = cluster_id self.fuel_web.update_nodes( cluster_id, config.get('nodes') ) self.fuel_web.verify_network(cluster_id) self.fuel_web.deploy_cluster_wait(cluster_id) self.fuel_web.verify_network(cluster_id) self.env.make_snapshot(snapshot_name, is_make=True) self.env.resume_environment() return True else: logger.error("Can't deploy cluster because snapshot" " with bootstrapped nodes didn't revert") raise RuntimeError("Can't deploy cluster because snapshot" " with bootstrapped nodes didn't revert")
def check_ipconfig_for_template(self, cluster_id, network_template, networks): logger.info("Checking that IP addresses configuration on nodes " "corresponds to used networking template...") # Network for Neutron is configured in namespaces (l3/dhcp agents) # and a bridge for it doesn't have IP, so skipping it for now skip_roles = {'neutron/private'} for node in self.fuel_web.client.list_cluster_nodes(cluster_id): node_networks = set() node_group_name = [ng['name'] for ng in self.fuel_web.client.get_nodegroups() if ng['id'] == node['group_id']][0] for role in node['roles']: node_networks.update( self.get_template_ep_for_role(template=network_template, role=role, nodegroup=node_group_name, skip_net_roles=skip_roles)) with self.env.d_env.get_ssh_to_remote(node['ip']) as remote: for network in networks: if network['name'] not in node_networks or \ network['group_id'] != node['group_id']: continue logger.debug('Checking interface "{0}" for IP network ' '"{1}" on "{2}"'.format(network['interface'], network['cidr'], node['hostname'])) self.check_interface_ip_exists(remote, network['interface'], network['cidr'])
def prepare_repository(self): """Prepare admin node to packages testing Scenario: 1. Temporary set nameserver to local router on admin node 2. Install tools to manage rpm/deb repository 3. Retrieve list of packages from custom repository 4. Download packages to local rpm/deb repository 5. Update .yaml file with new packages version 6. Re-generate repo using shell scripts on admin node """ # Check necessary settings and revert a snapshot if not self.custom_pkgs_mirror: return logger.info("Custom mirror with new packages: {0}" .format(settings.CUSTOM_PKGS_MIRROR)) if settings.OPENSTACK_RELEASE_UBUNTU in settings.OPENSTACK_RELEASE: # Ubuntu master_tools = ['dpkg', 'dpkg-devel', 'dpkg-dev'] self.install_tools(master_tools) self.get_pkgs_list_ubuntu() pkgs_local_path = ('{0}/pool/' .format(self.local_mirror_ubuntu)) self.download_pkgs(pkgs_local_path) self.regenerate_repo(self.ubuntu_script, self.local_mirror_ubuntu) else: # CentOS master_tools = ['createrepo'] self.install_tools(master_tools) self.get_pkgs_list_centos() pkgs_local_path = '{0}/Packages/'.format(self.local_mirror_centos) self.download_pkgs(pkgs_local_path) self.regenerate_repo(self.centos_script, self.local_mirror_centos)
def show_step(self, step, details='', initialize=False): """Show a description of the step taken from docstring :param int/str step: step number to show :param str details: additional info for a step """ test_func_name = get_test_method_name() if initialize or step == 1: self.current_log_step = step else: self.current_log_step += 1 if self.current_log_step != step: error_message = 'The step {} should be {} at {}' error_message = error_message.format( step, self.current_log_step, test_func_name ) logger.error(error_message) test_func = getattr(self.__class__, test_func_name) docstring = test_func.__doc__ docstring = '\n'.join([s.strip() for s in docstring.split('\n')]) steps = {s.split('. ')[0]: s for s in docstring.split('\n') if s and s[0].isdigit()} if details: details_msg = ': {0} '.format(details) else: details_msg = '' if str(step) in steps: logger.info("\n" + " " * 55 + "<<< {0} {1}>>>" .format(steps[str(step)], details_msg)) else: logger.info("\n" + " " * 55 + "<<< {0}. (no step description " "in scenario) {1}>>>".format(str(step), details_msg))
def regenerate_repo(self, regenerate_script, local_mirror_path): # Uploading scripts that prepare local repositories: # 'regenerate_centos_repo' and 'regenerate_ubuntu_repo' try: self.ssh_manager.upload_to_remote( ip=self.ip, source='{0}/{1}'.format(self.path_scripts, regenerate_script), target=self.remote_path_scripts ) self.ssh_manager.execute_on_remote( ip=self.ip, cmd='chmod 755 {0}/{1}'.format(self.remote_path_scripts, regenerate_script) ) except Exception: logger.error('Could not upload scripts for updating repositories.' '\n{0}'.format(traceback.format_exc())) raise # Update the local repository using previously uploaded script. script_cmd = '{0}/{1} {2} {3}'.format(self.remote_path_scripts, regenerate_script, local_mirror_path, self.ubuntu_release) script_result = self.ssh_manager.execute( ip=self.ip, cmd=script_cmd ) assert_equal(0, script_result['exit_code'], self.assert_msg(script_cmd, script_result['stderr'])) logger.info('Local repository {0} has been updated successfully.' .format(local_mirror_path))
def do_sync_time(self, ntps=None): # 0. 'ntps' can be filled by __init__() or outside the class self.ntps = ntps or self.ntps if not self.ntps: raise ValueError("No servers were provided to synchronize " "the time in self.ntps") # 1. Set actual time on all nodes via 'ntpdate' [ntp.set_actual_time() for ntp in self.ntps] assert_true(self.is_synchronized, "Time on nodes was not set:" " \n{0}".format(self.report_not_synchronized())) # 2. Restart NTPD service [ntp.stop() for ntp in self.ntps] [ntp.start() for ntp in self.ntps] # 3. Wait for established peers [ntp.wait_peer() for ntp in self.ntps] assert_true(self.is_connected, "Time on nodes was not synchronized:" " \n{0}".format(self.report_not_connected())) # 4. Report time on nodes for ntp in self.ntps: logger.info("Time on '{0}' = {1}".format(ntp.node_name, ntp.date()[0].rstrip()))
def download_pkgs(self, pkgs_local_path): # Process the packages list: total_pkgs = len(self.pkgs_list) logger.info('Found {0} custom package(s)'.format(total_pkgs)) for npkg, pkg in enumerate(self.pkgs_list): # TODO: Previous versions of the updating packages must be removed # to avoid unwanted packet manager dependencies resolution # (when some package still depends on other package which # is not going to be installed) logger.info('({0}/{1}) Downloading package: {2}/{3}' .format(npkg + 1, total_pkgs, self.custom_pkgs_mirror, pkg["filename:"])) pkg_ext = pkg["filename:"].split('.')[-1] if pkg_ext == 'deb': path_suff = 'main/' elif pkg_ext == 'udeb': path_suff = 'debian-installer/' else: path_suff = '' wget_cmd = "wget --no-verbose --directory-prefix {0} {1}/{2}"\ .format(pkgs_local_path + path_suff, self.custom_pkgs_mirror, pkg["filename:"]) wget_result = self.ssh_manager.execute( ip=self.ip, cmd=wget_cmd ) assert_equal(0, wget_result['exit_code'], self.assert_msg(wget_cmd, wget_result['stderr']))
def get_sha_sum(file_path): logger.debug('Get md5 fo file {0}'.format(file_path)) md5_sum = SSHManager().execute_on_remote( SSHManager().admin_ip, cmd='md5sum {0}'.format( file_path))['stdout_str'].strip() logger.info('MD5 is {0}'.format(md5_sum)) return md5_sum
def replace_rpm_package(package): """Replaced rpm package.rpm on master node with package.rpm from review """ ssh = SSHManager() logger.info("Patching {}".format(package)) if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') try: # Upload package target_path = '/var/www/nailgun/{}/'.format(package) ssh.upload_to_remote( ip=ssh.admin_ip, source=settings.UPDATE_FUEL_PATH.rstrip('/'), target=target_path) package_name = package package_ext = '*.noarch.rpm' pkg_path = os.path.join(target_path, '{}{}'.format(package_name, package_ext)) full_package_name = get_full_filename(wildcard_name=pkg_path) logger.debug('Package name is {0}'.format(full_package_name)) full_package_path = os.path.join(os.path.dirname(pkg_path), full_package_name) # Update package on master node if not does_new_pkg_equal_to_installed_pkg( installed_package=package_name, new_package=full_package_path): update_rpm(path=full_package_path) except Exception: logger.error("Could not upload package") raise
def _create_net_subnet(self, cluster): """Create net and subnet""" contrail_ip = self.fuel_web.get_public_vip(cluster) logger.info('The ip is %s', contrail_ip) net = Common( controller_ip=contrail_ip, user='******', password='******', tenant='admin' ) net.neutron.create_network(body={ 'network': { 'name': 'net04', 'admin_state_up': True, } }) network_id = '' network_dic = net.neutron.list_networks() for dd in network_dic['networks']: if dd.get("name") == "net04": network_id = dd.get("id") if network_id == "": logger.error('Network id empty') logger.debug("id {0} to master node".format(network_id)) net.neutron.create_subnet(body={ 'subnet': { 'network_id': network_id, 'ip_version': 4, 'cidr': '10.100.0.0/24', 'name': 'subnet04', } })
def update_connection(self, ip, login=None, password=None, keys=None, port=22): """Update existed connection :param ip: host ip string :param login: login string :param password: password string :param keys: list of keys :param port: ssh port int :return: None """ if (ip, port) in self.connections: logger.info('SSH_MANAGER:Close connection for {ip}:{port}'.format( ip=ip, port=port)) self.connections[(ip, port)].clear() logger.info('SSH_MANAGER:Create new connection for ' '{ip}:{port}'.format(ip=ip, port=port)) self.connections[(ip, port)] = SSHClient( host=ip, port=port, username=login, password=password, private_keys=keys if keys is not None else [] )
def ceph_rados_gw(self): """Deploy ceph HA with RadosGW for objects Scenario: 1. Create cluster with Neutron 2. Add 3 nodes with controller role 3. Add 3 nodes with compute and ceph-osd role 4. Deploy the cluster 5. Check ceph status 6. Run OSTF tests 7. Check the radosgw daemon is started Duration 90m Snapshot ceph_rados_gw """ def radosgw_started(remote): return remote.check_call('pkill -0 radosgw')['exit_code'] == 0 self.env.revert_snapshot("ready") self.env.bootstrap_nodes( self.env.d_env.nodes().slaves[:6]) cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=settings.DEPLOYMENT_MODE, settings={ 'volumes_lvm': False, 'volumes_ceph': True, 'images_ceph': True, 'objects_ceph': True, 'tenant': 'rados', 'user': '******', 'password': '******' } ) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller'], 'slave-02': ['controller'], 'slave-03': ['controller'], 'slave-04': ['compute', 'ceph-osd'], 'slave-05': ['compute', 'ceph-osd'], 'slave-06': ['compute', 'ceph-osd'] } ) self.fuel_web.verify_network(cluster_id) # Deploy cluster self.fuel_web.deploy_cluster_wait(cluster_id) # Network verification self.fuel_web.verify_network(cluster_id) # HAProxy backend checking controller_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( cluster_id, ['controller']) for node in controller_nodes: logger.info("Check all HAProxy backends on {}".format( node['meta']['system']['fqdn'])) haproxy_status = checkers.check_haproxy_backend(node['ip']) assert_equal(haproxy_status['exit_code'], 1, "HAProxy backends are DOWN. {0}".format( haproxy_status)) self.fuel_web.check_ceph_status(cluster_id) # Run ostf self.fuel_web.run_ostf(cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity']) # Check the radosgw daemon is started with self.fuel_web.get_ssh_for_node('slave-01') as remote: assert_true(radosgw_started(remote), 'radosgw daemon started') self.env.make_snapshot("ceph_rados_gw")
def get_md5sum(self, file_path, controller_ssh, vm_ip, creds=()): logger.info("Get file md5sum and compare it with previous one") out = self.execute_through_host( controller_ssh, vm_ip, "md5sum {:s}".format(file_path), creds) return out['stdout']
def ceph_ha_one_controller_with_cinder(self): """Deploy ceph with cinder in ha mode with 1 controller Scenario: 1. Create cluster 2. Add 1 node with controller role 3. Add 1 node with compute role 4. Add 2 nodes with cinder and ceph OSD roles 5. Deploy the cluster 6. Check ceph status 7. Check partitions on controller node Duration 40m Snapshot ceph_ha_one_controller_with_cinder """ try: self.check_run('ceph_ha_one_controller_with_cinder') except SkipTest: return self.env.revert_snapshot("ready") self.env.bootstrap_nodes( self.env.d_env.nodes().slaves[:4]) cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=settings.DEPLOYMENT_MODE, settings={ 'volumes_ceph': False, 'images_ceph': True, 'osd_pool_size': '2', 'volumes_lvm': True, 'tenant': 'ceph2', 'user': '******', 'password': '******' } ) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller'], 'slave-02': ['compute'], 'slave-03': ['cinder', 'ceph-osd'], 'slave-04': ['cinder', 'ceph-osd'] } ) # Cluster deploy self.fuel_web.deploy_cluster_wait(cluster_id) self.fuel_web.check_ceph_status(cluster_id) disks = self.fuel_web.client.get_node_disks( self.fuel_web.get_nailgun_node_by_name('slave-01')['id']) logger.info("Current disk partitions are: \n{d}".format(d=disks)) logger.info("Check unallocated space") # We expect failure here only for release 5.0 due to bug # https://bugs.launchpad.net/fuel/+bug/1306625, so it is # necessary to assert_true in the next release. assert_false( checkers.check_unallocated_space(disks, contr_img_ceph=True), "Check unallocated space on controller") # Run ostf self.fuel_web.run_ostf(cluster_id=cluster_id) self.env.make_snapshot("ceph_ha_one_controller_with_cinder", is_make=True)
def check_rh_hard_reboot(self): """Check that resumed VM is working properly after hard reboot of RH-based compute Scenario: 1. Revert environment with RH-compute. 2. Check that services are ready. 3. Boot VM on compute and check its connectivity via floating ip. 4. Hard reboot RH-based compute. 5. Verify VM connectivity via floating ip after successful reboot and VM resume action. Duration 20m Snapshot check_rh_hard_reboot """ self.show_step(1, initialize=True) self.env.revert_snapshot('ready_ha_one_controller_with_rh_compute', skip_timesync=True, skip_slaves_check=True) self.check_slaves_are_ready() logger.debug('All slaves online.') self.show_step(2) cluster_id = self.fuel_web.get_last_created_cluster() os_conn = os_actions.OpenStackActions( self.fuel_web.get_public_vip(cluster_id)) self.fuel_web.assert_cluster_ready(os_conn, smiles_count=5) logger.debug('Cluster up and ready.') self.show_step(3) cluster_id = self.fuel_web.get_last_created_cluster() controllers = self.fuel_web.get_nailgun_cluster_nodes_by_roles( cluster_id, roles=('controller', )) os_conn = os_actions.OpenStackActions( self.fuel_web.get_public_vip(cluster_id)) asserts.assert_equal( len(controllers), 1, 'Environment does not have 1 controller node, ' 'found {} nodes!'.format(len(controllers))) compute = self.fuel_web.get_nailgun_cluster_nodes_by_roles( cluster_id, ['compute'])[0] target_node = self.fuel_web.get_devops_node_by_nailgun_node(compute) target_node_ip = self.fuel_web.get_node_ip_by_devops_name( target_node.name) net_label = self.fuel_web.get_cluster_predefined_networks_name( cluster_id)['private_net'] vm = os_conn.create_server_for_migration(neutron=True, label=net_label) vm_floating_ip = os_conn.assign_floating_ip(vm) logger.info('Trying to get vm via tcp.') try: wait(lambda: tcp_ping(vm_floating_ip.ip, 22), timeout=120) except TimeoutError: raise TimeoutError('Can not ping instance' ' by floating ip {0}'.format(vm_floating_ip.ip)) logger.info('VM is accessible via ip: {0}'.format(vm_floating_ip.ip)) self.show_step(4) target_node.destroy() asserts.assert_false(target_node.driver.node_active(node=target_node), 'Target node still active') target_node.start() asserts.assert_true(target_node.driver.node_active(node=target_node), 'Target node did not start') self.wait_for_slave_provision(target_node_ip) self.fuel_web.assert_cluster_ready(os_conn, smiles_count=5) logger.info('All cluster services up and ' 'running after compute hard reboot.') self.show_step(5) asserts.assert_equal( os_conn.get_instance_detail(vm).status, "ACTIVE", "Instance did not reach active state after compute back online, " "current state is {0}".format( os_conn.get_instance_detail(vm).status)) logger.info('Spawned VM is ACTIVE. Trying to ' 'access it via ip: {0}'.format(vm_floating_ip.ip)) try: wait(lambda: tcp_ping(vm_floating_ip.ip, 22), timeout=120) except TimeoutError: raise TimeoutError('Can not ping instance' ' by floating ip {0}'.format(vm_floating_ip.ip)) logger.info('VM is accessible. Deleting it.') os_conn.delete_instance(vm) os_conn.verify_srv_deleted(vm)
def admin_install_updates(self): """Update packages using yum and install updates via update-master-node.sh tool""" logger.info('Searching for updates..') update_command = 'yum clean expire-cache && ' \ 'yum update -y 2>>/var/log/yum-update-error.log' logger.info('Performing yum clean and update commands') update_result = self.ssh_manager.check_call( ip=self.ssh_manager.admin_ip, command=update_command, error_info='Packages update failed, inspect logs for details') logger.info('Packages were updated successfully') # Check if any packets were updated and update was successful match_updated_count = re.search(r'Upgrade\s+(\d+)\s+Package', update_result['stdout_str']) # In case of package replacement, the new one is marked as # installed and the old one as removed match_installed_count = re.search(r'Install\s+(\d+)\s+Package', update_result['stdout_str']) match_complete_message = re.search(r'Complete!', update_result['stdout_str']) match_no_updates = re.search("No Packages marked for Update", update_result['stdout_str']) if match_no_updates or not match_complete_message \ or not (match_updated_count or match_installed_count): logger.warning('No updates were found or update was incomplete.') return updates_count = 0 if match_updated_count: updates_count += int(match_updated_count.group(1)) if match_installed_count: updates_count += int(match_installed_count.group(1)) logger.info('{0} package(s) were updated'.format(updates_count)) logger.info('Applying updates via update-master-node.sh') # LP #1664635 - we need to redirect stdout to /dev/null to avoid # ssh connection hanging on massive output from puppet run. cmd = '/usr/share/fuel-utils/update-master-node.sh > /dev/null 2>&1' self.ssh_manager.check_call( ip=self.ssh_manager.admin_ip, command=cmd, error_info='Update failed, inspect logs for details', ) logger.info('Update successful')
def setup_customisation(self): logger.info('Installing custom packages/manifests ' 'before master node bootstrap...')
def contrail_ha_with_shutdown_contrail_node(self): """Verify HA with deleting Contrail roles Scenario: 1. Create an environment with "Neutron with tunneling segmentation" as a network configuration 2. Enable and configure Contrail plugin 3. Add some controller, compute and storage nodes 4. Add 4 nodes with "contrail-db", "contarail-config" and "contrail-control" roles 5. Deploy cluster 6. Run OSTF tests 7. Check Controller and Contrail nodes status 8. Shutdown node with 'contrail-db', "contarail-config" and "contrail-control" roles 9. Deploy changes 10. Run OSTF tests 11. Check Controller and Contrail nodes status """ plugin.prepare_contrail_plugin(self, slaves=9) plugin.activate_plugin(self) # enable plugin in contrail settings vsrx_setup_result = plugin.activate_vsrx() # activate vSRX image conf_no_contrail = { 'slave-01': ['controller'], 'slave-02': ['controller'], 'slave-03': ['controller'], 'slave-04': ['compute'], 'slave-05': ['cinder'], # Here slave-06 with contrail 'slave-07': ['contrail-db', 'contrail-config', 'contrail-control'], 'slave-08': ['contrail-db', 'contrail-config', 'contrail-control'], 'slave-09': ['contrail-db', 'contrail-config', 'contrail-control'], } conf_contrail = { 'slave-06': ['contrail-db', 'contrail-config', 'contrail-control'] } def check_node_state(cluster_id, node_name, node_state): """Checks node state by it's name""" for node in self.fuel_web.client.list_cluster_nodes(cluster_id): if node_name in node['name']: assert_equal( node['status'], node_state, 'Nailgun node status is not %s but %s' % (node_state, node['status'])) # Deploy cluster and run OSTF openstack.update_deploy_check(self, dict(conf_no_contrail, **conf_contrail), is_vsrx=vsrx_setup_result) # Check all nodes are 'ready' for node_name in dict(conf_no_contrail, **conf_contrail): check_node_state(self.cluster_id, node_name, 'ready') # Shutdown contrail node for node in self.fuel_web.client.list_cluster_nodes(self.cluster_id): if 'slave-06' in node['name']: logger.info('Shutdown node "%s"' % node['name']) self.fuel_web.warm_shutdown_nodes( self.fuel_web.get_devops_nodes_by_nailgun_nodes([node])) break # Run OSTF tests again if vsrx_setup_result: self.fuel_web.run_ostf(cluster_id=self.cluster_id) # Check controller and contrail nodes states node_roles = {'controller', 'contrail-config'} for node_name, roles in conf_no_contrail.items(): if node_roles & set(roles): check_node_state(self.cluster_id, node_name, 'ready')
def create_image(self, **kwargs): image = self.glance.images.create(**kwargs) logger.info("Created image: '{0}'".format(image.id)) logger.info("Image status: '{0}'".format(image.status)) return image
def dvs_regression(self): """Deploy cluster with plugin and vmware datastore backend. Scenario: 1. Revert to dvs_bvt snapshot. 2. Create non default network net_1. 3. Launch instances with created network in nova and vcenter az. 4. Create Security groups. 5. Attached created security groups to instances. 6. Check connection between instances from different az. Duration: 1.8 hours """ self.show_step(1) self.env.revert_snapshot("dvs_bvt") cluster_id = self.fuel_web.get_last_created_cluster() os_ip = self.fuel_web.get_public_vip(cluster_id) os_conn = os_actions.OpenStackActions(os_ip, SERVTEST_USERNAME, SERVTEST_PASSWORD, SERVTEST_TENANT) tenant = os_conn.get_tenant(SERVTEST_TENANT) # Create non default network with subnet self.show_step(2) logger.info('Create network {}'.format(self.net_data[0].keys()[0])) net_1 = os_conn.create_network(network_name=self.net_data[0].keys()[0], tenant_id=tenant.id)['network'] subnet = os_conn.create_subnet( subnet_name=net_1['name'], network_id=net_1['id'], cidr=self.net_data[0][self.net_data[0].keys()[0]], ip_version=4) # Check that network are created assert_true(os_conn.get_network(net_1['name'])['id'] == net_1['id']) # Add net_1 to default router router = os_conn.get_router(os_conn.get_network(self.ext_net_name)) os_conn.add_router_interface(router_id=router["id"], subnet_id=subnet["id"]) self.show_step(3) # Launch 2 vcenter VMs and 2 nova VMs in the tenant network net_01 openstack.create_instances(os_conn=os_conn, vm_count=1, nics=[{ 'net-id': net_1['id'] }]) # Launch 2 vcenter VMs and 2 nova VMs in the default network net_1 = os_conn.nova.networks.find(label=self.inter_net_name) instances = openstack.create_instances(os_conn=os_conn, vm_count=1, nics=[{ 'net-id': net_1.id }]) openstack.verify_instance_state(os_conn) self.show_step(4) # Create security groups SG_1 to allow ICMP traffic. # Add Ingress rule for ICMP protocol to SG_1 # Create security groups SG_2 to allow TCP traffic 22 port. # Add Ingress rule for TCP protocol to SG_2 sec_name = ['SG1', 'SG2'] sg1 = os_conn.nova.security_groups.create(sec_name[0], "descr") sg2 = os_conn.nova.security_groups.create(sec_name[1], "descr") rulesets = [ { # ssh 'ip_protocol': 'tcp', 'from_port': 22, 'to_port': 22, 'cidr': '0.0.0.0/0', }, { # ping 'ip_protocol': 'icmp', 'from_port': -1, 'to_port': -1, 'cidr': '0.0.0.0/0', } ] os_conn.nova.security_group_rules.create(sg1.id, **rulesets[0]) os_conn.nova.security_group_rules.create(sg2.id, **rulesets[1]) # Remove default security group and attach SG_1 and SG2 to VMs self.show_step(5) srv_list = os_conn.get_servers() for srv in srv_list: srv.remove_security_group(srv.security_groups[0]['name']) srv.add_security_group(sg1.id) srv.add_security_group(sg2.id) fip = openstack.create_and_assign_floating_ips(os_conn, instances) # Check ping between VMs self.show_step(6) ip_pair = dict.fromkeys(fip) for key in ip_pair: ip_pair[key] = [value for value in fip if key != value] openstack.check_connection_vms(ip_pair)
def migrate_vm_backed_with_ceph(self): """Check VM backed with ceph migration in ha mode with 1 controller Scenario: 1. Create cluster 2. Add 1 node with controller and ceph OSD roles 3. Add 2 nodes with compute and ceph OSD roles 4. Deploy the cluster 5. Check ceph status 6. Run OSTF 7. Create a new VM, assign floating ip 8. Migrate VM 9. Check cluster and server state after migration 10. Terminate VM 11. Check that DHCP lease is not offered for MAC of deleted VM 12. Create a new VM for migration, assign floating ip 13. Create a volume and attach it to the VM 14. Create filesystem on the new volume and mount it to the VM 15. Migrate VM 16. Mount the volume after migration 17. Check cluster and server state after migration 18. Terminate VM Duration 35m Snapshot vm_backed_with_ceph_live_migration """ self.env.revert_snapshot("ready_with_3_slaves") self.show_step(1) cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=settings.DEPLOYMENT_MODE, settings={ 'volumes_ceph': True, 'images_ceph': True, 'ephemeral_ceph': True, 'volumes_lvm': False, 'net_provider': 'neutron', 'net_segment_type': NEUTRON_SEGMENT_TYPE, }) self.show_step(2) self.show_step(3) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller', 'ceph-osd'], 'slave-02': ['compute', 'ceph-osd'], 'slave-03': ['compute', 'ceph-osd'] }) creds = ("cirros", "test") self.show_step(4) # Cluster deploy self.fuel_web.deploy_cluster_wait(cluster_id) def _check(): # Run volume test several times with hope that it pass test_path = map_ostf.OSTF_TEST_MAPPING.get( 'Create volume and attach it to instance') logger.debug('Start to run test {0}'.format(test_path)) self.fuel_web.run_single_ostf_test(cluster_id, test_sets=['smoke'], test_name=test_path) self.show_step(5) try: _check() except AssertionError: logger.debug(AssertionError) logger.debug("Test failed from first probe," " we sleep 60 second try one more time " "and if it fails again - test will fails ") time.sleep(60) _check() self.show_step(6) # Run ostf self.fuel_web.run_ostf(cluster_id) self.show_step(7) # Create new server os = os_actions.OpenStackActions( self.fuel_web.get_public_vip(cluster_id)) logger.info("Create new server") srv = os.create_server_for_migration( neutron=True, scenario='./fuelweb_test/helpers/instance_initial_scenario') logger.info("Srv is currently in status: %s" % srv.status) # Prepare to DHCP leases checks srv_instance_ip = os.get_nova_instance_ip(srv, net_name='net04') srv_host_name = self.fuel_web.find_devops_node_by_nailgun_fqdn( os.get_srv_hypervisor_name(srv), self.env.d_env.nodes().slaves[:3]).name net_id = os.get_network('net04')['id'] ports = os.get_neutron_dhcp_ports(net_id) dhcp_server_ip = ports[0]['fixed_ips'][0]['ip_address'] with self.fuel_web.get_ssh_for_node(srv_host_name) as srv_remote_node: srv_instance_mac = os.get_instance_mac(srv_remote_node, srv) logger.info("Assigning floating ip to server") floating_ip = os.assign_floating_ip(srv) srv_host = os.get_srv_host_name(srv) logger.info("Server is on host %s" % srv_host) wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120) with self.fuel_web.get_ssh_for_node("slave-01") as remote: md5before = os.get_md5sum("/home/test_file", remote, floating_ip.ip, creds) self.show_step(8) logger.info("Get available computes") avail_hosts = os.get_hosts_for_migr(srv_host) logger.info("Migrating server") new_srv = os.migrate_server(srv, avail_hosts[0], timeout=200) logger.info("Check cluster and server state after migration") wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120) with self.fuel_web.get_ssh_for_node("slave-01") as remote: md5after = os.get_md5sum("/home/test_file", remote, floating_ip.ip, creds) assert_true( md5after in md5before, "Md5 checksums don`t match." "Before migration md5 was equal to: {bef}" "Now it eqals: {aft}".format(bef=md5before, aft=md5after)) self.show_step(9) with self.fuel_web.get_ssh_for_node("slave-01") as remote: res = os.execute_through_host( remote, floating_ip.ip, "ping -q -c3 -w10 {0} | grep 'received' |" " grep -v '0 packets received'".format( settings.PUBLIC_TEST_IP), creds) logger.info("Ping {0} result on vm is: {1}".format( settings.PUBLIC_TEST_IP, res['stdout'])) logger.info("Check Ceph health is ok after migration") self.fuel_web.check_ceph_status(cluster_id) logger.info("Server is now on host %s" % os.get_srv_host_name(new_srv)) self.show_step(10) logger.info("Terminate migrated server") os.delete_instance(new_srv) assert_true(os.verify_srv_deleted(new_srv), "Verify server was deleted") self.show_step(11) # Check if the dhcp lease for instance still remains # on the previous compute node. Related Bug: #1391010 with self.fuel_web.get_ssh_for_node('slave-01') as remote: dhcp_port_tag = ovs_get_tag_by_port(remote, ports[0]['id']) assert_false( checkers.check_neutron_dhcp_lease(remote, srv_instance_ip, srv_instance_mac, dhcp_server_ip, dhcp_port_tag), "Instance has been deleted, but it's DHCP lease " "for IP:{0} with MAC:{1} still offers by Neutron DHCP" " agent.".format(srv_instance_ip, srv_instance_mac)) self.show_step(12) # Create a new server logger.info("Create a new server for migration with volume") srv = os.create_server_for_migration( neutron=True, scenario='./fuelweb_test/helpers/instance_initial_scenario') logger.info("Srv is currently in status: %s" % srv.status) logger.info("Assigning floating ip to server") floating_ip = os.assign_floating_ip(srv) srv_host = os.get_srv_host_name(srv) logger.info("Server is on host %s" % srv_host) self.show_step(13) logger.info("Create volume") vol = os.create_volume() logger.info("Attach volume to server") os.attach_volume(vol, srv) self.show_step(14) wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120) logger.info("Create filesystem and mount volume") with self.fuel_web.get_ssh_for_node("slave-01") as remote: os.execute_through_host(remote, floating_ip.ip, 'sudo sh /home/mount_volume.sh', creds) os.execute_through_host(remote, floating_ip.ip, 'sudo touch /mnt/file-on-volume', creds) self.show_step(15) logger.info("Get available computes") avail_hosts = os.get_hosts_for_migr(srv_host) logger.info("Migrating server") new_srv = os.migrate_server(srv, avail_hosts[0], timeout=120) logger.info("Check cluster and server state after migration") wait(lambda: tcp_ping(floating_ip.ip, 22), timeout=120) self.show_step(16) logger.info("Mount volume after migration") with self.fuel_web.get_ssh_for_node("slave-01") as remote: out = os.execute_through_host(remote, floating_ip.ip, 'sudo mount /dev/vdb /mnt', creds) logger.info("out of mounting volume is: %s" % out['stdout']) with self.fuel_web.get_ssh_for_node("slave-01") as remote: out = os.execute_through_host(remote, floating_ip.ip, "sudo ls /mnt", creds) assert_true("file-on-volume" in out['stdout'], "File is abscent in /mnt") self.show_step(17) logger.info("Check Ceph health is ok after migration") self.fuel_web.check_ceph_status(cluster_id) logger.info("Server is now on host %s" % os.get_srv_host_name(new_srv)) self.show_step(18) logger.info("Terminate migrated server") os.delete_instance(new_srv) assert_true(os.verify_srv_deleted(new_srv), "Verify server was deleted") self.env.make_snapshot("vm_backed_with_ceph_live_migration")
def remove_controllers(self): """Deploy cluster with 3 controllers, remove 2 controllers and re-deploy, check hosts and corosync Scenario: 1. Create cluster 2. Add 3 controller, 1 compute 3. Deploy the cluster 4. Remove 2 controllers 5. Deploy changes 6. Run OSTF 7. Verify networks 8. Check /etc/hosts that removed nodes aren't present 9. Check corosync.conf that removed nodes aren't present Duration 120m Snapshot remove_controllers """ self.env.revert_snapshot("ha_scale_group_2_cluster") self.show_step(1, initialize=True) self.show_step(2) self.show_step(3) cluster_id = self.fuel_web.get_last_created_cluster() hosts = [] for node_name in ('slave-02', 'slave-03'): node = self.fuel_web.get_nailgun_node_by_devops_node( self.env.d_env.get_node(name=node_name)) hostname = ''.join( self.ssh_manager.check_call( ip=node['ip'], command="hostname")['stdout']).strip() hosts.append(hostname) logger.debug('hostname are {}'.format(hosts)) nodes = {'slave-02': ['controller'], 'slave-03': ['controller']} self.show_step(4) self.fuel_web.update_nodes(cluster_id, nodes, False, True) self.show_step(5) self.fuel_web.deploy_cluster_wait(cluster_id) self.show_step(6) self.fuel_web.run_ostf(cluster_id=cluster_id) self.show_step(7) self.fuel_web.verify_network(cluster_id) node = self.fuel_web.get_nailgun_node_by_devops_node( self.env.d_env.get_node(name='slave-01')) for host in hosts: self.show_step(8, initialize=True) cmd = "grep '{}' /etc/hosts".format(host) logger.info('Checking hosts on {}'.format(host)) result = self.ssh_manager.check_call(ip=node['ip'], command=cmd, expected=[1]) assert_equal(result['exit_code'], 1, "host {} is present in /etc/hosts".format(host)) self.show_step(9) cmd = "grep '{}' /etc/corosync/corosync.conf".format(host) logger.info('Checking corosync.conf on {}'.format(host)) result = self.ssh_manager.check_call(ip=node['ip'], command=cmd, expected=[1]) assert_equal( result['exit_code'], 1, "host {} is present in" " /etc/corosync/corosync.conf".format(host)) self.env.make_snapshot("remove_controllers")
def check_ceph_partitions_after_reboot(self): """Check that Ceph OSD partitions are remounted after reboot Scenario: 1. Create cluster in Ha mode with 1 controller 2. Add 1 node with controller role 3. Add 1 node with compute and Ceph OSD roles 4. Add 1 node with Ceph OSD role 5. Deploy the cluster 6. Check Ceph status 7. Read current partitions 8. Warm-reboot Ceph nodes 9. Read partitions again 10. Check Ceph health 11. Cold-reboot Ceph nodes 12. Read partitions again 13. Check Ceph health Duration 40m Snapshot check_ceph_partitions_after_reboot """ self.env.revert_snapshot("ready_with_3_slaves") self.show_step(1) cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=settings.DEPLOYMENT_MODE, settings={ 'volumes_ceph': True, 'images_ceph': True, 'ephemeral_ceph': True, 'volumes_lvm': False, 'net_provider': 'neutron', 'net_segment_type': NEUTRON_SEGMENT_TYPE, }) self.show_step(2) self.show_step(3) self.show_step(4) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller'], 'slave-02': ['compute', 'ceph-osd'], 'slave-03': ['ceph-osd'] }) self.show_step(5) # Deploy cluster self.fuel_web.deploy_cluster_wait(cluster_id) self.show_step(6) for node in ["slave-02", "slave-03"]: self.show_step(7, node) logger.info("Get partitions for {node}".format(node=node)) _ip = self.fuel_web.get_nailgun_node_by_name(node)['ip'] with self.env.d_env.get_ssh_to_remote(_ip) as remote: before_reboot_partitions = [ checkers.get_ceph_partitions(remote, "/dev/vd{p}".format(p=part)) for part in ["b", "c"] ] self.show_step(8, node) logger.info("Warm-restart nodes") self.fuel_web.warm_restart_nodes( [self.fuel_web.environment.d_env.get_node(name=node)]) self.show_step(9, node) logger.info( "Get partitions for {node} once again".format(node=node)) _ip = self.fuel_web.get_nailgun_node_by_name(node)['ip'] with self.env.d_env.get_ssh_to_remote(_ip) as remote: after_reboot_partitions = [ checkers.get_ceph_partitions(remote, "/dev/vd{p}".format(p=part)) for part in ["b", "c"] ] if before_reboot_partitions != after_reboot_partitions: logger.info("Partitions don`t match") logger.info("Before reboot: %s" % before_reboot_partitions) logger.info("After reboot: %s" % after_reboot_partitions) raise Exception() self.show_step(10, node) logger.info("Check Ceph health is ok after reboot") self.fuel_web.check_ceph_status(cluster_id) self.show_step(11, node) logger.info("Cold-restart nodes") self.fuel_web.cold_restart_nodes( [self.fuel_web.environment.d_env.get_node(name=node)]) self.show_step(12, node) _ip = self.fuel_web.get_nailgun_node_by_name(node)['ip'] with self.env.d_env.get_ssh_to_remote(_ip) as remote: after_reboot_partitions = [ checkers.get_ceph_partitions(remote, "/dev/vd{p}".format(p=part)) for part in ["b", "c"] ] if before_reboot_partitions != after_reboot_partitions: logger.info("Partitions don`t match") logger.info("Before reboot: %s" % before_reboot_partitions) logger.info("After reboot: %s" % after_reboot_partitions) raise Exception() self.show_step(13, node) logger.info("Check Ceph health is ok after reboot") self.fuel_web.check_ceph_status(cluster_id)
def delete_custom_nodegroup(self): """Delete nodegroup, check its nodes are marked as 'error' Scenario: 1. Revert snapshot with cluster with nodes in custom nodegroup 2. Save cluster network configuration 3. Reset cluster 4. Remove custom nodegroup 5. Check nodes from custom nodegroup have 'error' status 6. Re-create custom nodegroup and upload saved network configuration 7. Assign 'error' nodes to new nodegroup 8. Check nodes from custom nodegroup are in 'discover' state Duration 30m """ self.show_step(1, initialize=True) self.env.revert_snapshot('deploy_controllers_from_custom_nodegroup') cluster_id = self.fuel_web.get_last_created_cluster() self.fuel_web.assert_nodes_in_ready_state(cluster_id) self.show_step(2) network_config = self.fuel_web.client.get_networks(cluster_id) self.show_step(3) custom_nodes = self.env.d_env.nodes().slaves[3:6] self.fuel_web.stop_reset_env_wait(cluster_id) # TODO(apanchenko): remove sleep(181) workaround when the issue with # TODO(apanchenko): cluster reset is fixed (see LP#1588193) # Nailgun waits 180 seconds before marking slave node as offline time.sleep(181) logger.info('Waiting for all nodes online for 900 seconds...') wait(lambda: all(n['online'] for n in self.fuel_web.client. list_cluster_nodes(cluster_id)), timeout=15 * 60, timeout_msg='Timeout while waiting nodes to become online ' 'after reset') self.show_step(4) custom_nodegroup = [ ng for ng in self.fuel_web.client.get_nodegroups() if ng['name'] == NODEGROUPS[1]['name'] ][0] self.fuel_web.client.delete_nodegroup(custom_nodegroup['id']) self.show_step(5) logger.info('Wait all nodes from custom nodegroup become ' 'in error state..') for slave in custom_nodes: # pylint: disable=undefined-loop-variable wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(slave)[ 'status'] == 'error', timeout=60 * 5, timeout_msg='Node {} status wasn\'t changed ' 'to "error"!'.format(slave.name)) # pylint: enable=undefined-loop-variable logger.info('Node {} is in "error" state'.format(slave.name)) self.show_step(6) new_nodegroup = self.fuel_web.client.create_nodegroup( cluster_id, NODEGROUPS[1]['name']) logger.debug('Updating custom nodegroup ID in network configuration..') network_config_new = self.fuel_web.client.get_networks(cluster_id) for network in network_config['networks']: if network['group_id'] == custom_nodegroup['id']: network['group_id'] = new_nodegroup['id'] for new_network in network_config_new['networks']: if new_network['name'] == network['name'] and \ new_network['group_id'] == network['group_id']: network['id'] = new_network['id'] self.fuel_web.client.update_network( cluster_id, network_config['networking_parameters'], network_config['networks']) self.show_step(7) self.fuel_web.client.assign_nodegroup(new_nodegroup['id'], [ self.fuel_web.get_nailgun_node_by_devops_node(node) for node in custom_nodes ]) self.show_step(8) logger.info('Wait all nodes from custom nodegroup become ' 'in discover state..') for slave in custom_nodes: wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(slave)[ 'status'] == 'discover', timeout=60 * 5, timeout_msg='Node {} status wasn\'t changed ' 'to "discover"!'.format(slave.name)) logger.info('Node {} is in "discover" state'.format(slave.name)) self.env.make_snapshot("delete_custom_nodegroup")
def compute_stop_reinstallation(self): """Verify stop reinstallation of compute. Scenario: 1. Revert the snapshot 2. Create an OS volume and OS instance 3. Mark 'cinder' and 'vm' partitions to be preserved 4. Stop reinstallation process of compute 5. Start the reinstallation process again 6. Run network verification 7. Run OSTF 8. Verify that the volume is present and has 'available' status after the node reinstallation 9. Verify that the VM is available and pingable after the node reinstallation Duration: 115m """ self.env.revert_snapshot("node_reinstallation_env") cluster_id = self.fuel_web.get_last_created_cluster() # Create an OS volume os_conn = os_actions.OpenStackActions( self.fuel_web.get_public_vip(cluster_id)) volume = os_conn.create_volume() # Create an OS instance cmp_host = os_conn.get_hypervisors()[0] net_label = self.fuel_web.get_cluster_predefined_networks_name( cluster_id)['private_net'] vm = os_conn.create_server_for_migration( neutron=True, availability_zone="nova:{0}".format(cmp_host.hypervisor_hostname), label=net_label) vm_floating_ip = os_conn.assign_floating_ip(vm) devops_helpers.wait( lambda: devops_helpers.tcp_ping(vm_floating_ip.ip, 22), timeout=120) cmp_nailgun = self.fuel_web.get_nailgun_node_by_fqdn( cmp_host.hypervisor_hostname) # Mark 'cinder' and 'vm' partitions to be preserved with self.env.d_env.get_admin_remote() as remote: preserve_partition(remote, cmp_nailgun['id'], "cinder") preserve_partition(remote, cmp_nailgun['id'], "vm") slave_nodes = self.fuel_web.client.list_cluster_nodes(cluster_id) devops_nodes = self.fuel_web.get_devops_nodes_by_nailgun_nodes( slave_nodes) logger.info('Stop reinstallation process') self._stop_reinstallation(self.fuel_web, cluster_id, [str(cmp_nailgun['id'])], devops_nodes) self.fuel_web.verify_network(cluster_id) logger.info('Start the reinstallation process again') NodeReinstallationEnv.reinstall_nodes(self.fuel_web, cluster_id, [str(cmp_nailgun['id'])]) self.fuel_web.verify_network(cluster_id) self.fuel_web.run_ostf(cluster_id, test_sets=['ha', 'smoke', 'sanity']) # Verify that the created volume is still available try: volume = os_conn.cinder.volumes.get(volume.id) except NotFound: raise AssertionError( "{0} volume is not available after its {1} hosting node " "reinstallation".format(volume.id, cmp_nailgun['fqdn'])) expected_status = "available" assert_equal( expected_status, volume.status, "{0} volume status is {1} after its {2} hosting node " "reinstallation. Expected status is {3}.".format( volume.id, volume.status, cmp_nailgun['fqdn'], expected_status)) # Verify that the VM is still available try: os_conn.verify_instance_status(vm, 'ACTIVE') except AssertionError: raise AssertionError( "{0} VM is not available after its {1} hosting node " "reinstallation".format(vm.name, cmp_host.hypervisor_hostname)) assert_true( devops_helpers.tcp_ping(vm_floating_ip.ip, 22), "{0} VM is not accessible via its {1} floating " "ip".format(vm.name, vm_floating_ip))
def deploy_controllers_from_custom_nodegroup(self): """Assigning controllers to non-default nodegroup Scenario: 1. Revert snapshot with ready master node 2. Create environment with Neutron VXLAN and custom nodegroup 3. Configure network floating ranges to use public network from custom nodegroup 4. Bootstrap slaves from custom nodegroup 5. Bootstrap slave nodes from default nodegroup 6. Add 3 nodes from 'custom' nodegroup as controllers Add 2 nodes from 'default' nodegroup as compute and cinder 7. Run network verification 8. Deploy environment 9. Run network verification 10. Run OSTF 11. Check addresses allocated for VIPs belong to networks from custom nodegroup Duration 120m Snapshot deploy_controllers_from_custom_nodegroup """ if not MULTIPLE_NETWORKS: raise SkipTest('MULTIPLE_NETWORKS not enabled') self.show_step(1, initialize=True) self.check_run("deploy_controllers_from_custom_nodegroup") self.env.revert_snapshot("ready") self.show_step(2) cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__, mode=DEPLOYMENT_MODE_HA, settings={ "net_provider": 'neutron', "net_segment_type": NEUTRON_SEGMENT['tun'] }, configure_ssl=False) self.show_step(3) # floating range public2_cidr = self.env.d_env.get_network(name='public2').ip new_settings_float = { 'floating_ranges': [[ str(public2_cidr[len(public2_cidr) // 2]), str(public2_cidr[-2]) ]] } self.fuel_web.client.update_network(cluster_id, new_settings_float) self.show_step(4) custom_nodes = self.env.d_env.nodes().slaves[3:6] self.env.bootstrap_nodes(custom_nodes) # nodes 4, 5 and 6 self.show_step(5) default_nodes = self.env.d_env.nodes().slaves[0:2] self.env.bootstrap_nodes(default_nodes) # nodes 1 and 2 self.show_step(6) default_nodegroup = NODEGROUPS[0]['name'] custom_nodegroup = NODEGROUPS[1]['name'] self.fuel_web.update_nodes( cluster_id, { 'slave-04': [['controller'], custom_nodegroup], 'slave-05': [['controller'], custom_nodegroup], 'slave-06': [['controller'], custom_nodegroup], 'slave-01': [['compute'], default_nodegroup], 'slave-02': [['cinder'], default_nodegroup] }) # configuring ssl after nodes added to cluster due to vips in custom ng self.fuel_web.ssl_configure(cluster_id) self.show_step(7) self.fuel_web.verify_network(cluster_id) self.show_step(8) self.fuel_web.deploy_cluster_wait(cluster_id, timeout=150 * 60) self.show_step(9) self.fuel_web.verify_network(cluster_id) self.show_step(10) self.fuel_web.run_ostf(cluster_id=cluster_id) self.show_step(11) current_settings = self.fuel_web.client.get_networks(cluster_id) check = { 'vrouter_pub': 'public2', 'management': 'management2', 'public': 'public2', 'vrouter': 'management2' } for k in check: vip = netaddr.IPAddress(str(current_settings['vips'][k]['ipaddr'])) custom_net = netaddr.IPNetwork( str(self.env.d_env.get_network(name=check[k]).ip)) asserts.assert_true( vip in custom_net, '{0} is not from {1} network'.format(k, check[k])) logger.info('{0} is from {1} network'.format(k, check[k])) self.env.make_snapshot("deploy_controllers_from_custom_nodegroup", is_make=True)
def deploy_with_redeploy_and_modify_settings(self): """Deploy iteratively clusters from config, modify settings, redeploy Scenario: 1. Load clusters' configurations from the file 2. Revert snapshot with appropriate nodes count 3. Create a cluster from config 4. Update nodes accordingly to the config 5. Deploy the cluster 6. Run OSTF 7. Get cluster attributes 8. Modify randomly cluster attributes 9. Add if it's needed ceph nodes 10. Update cluster attributes with changed one 11. Redeploy cluster 12. Run OSTF 13. Go to the next config Duration xxx m Snapshot will be made for all failed configurations """ fail_trigger = False failed_confs = [] self.show_step(1) for conf in self.load_config('cluster_configs.yaml'): logger.info("Creating cluster from config with name: {}".format( conf['name'])) self.show_step(2, details=conf['name'], initialize=True) self.revert_snapshot(len(conf['nodes'])) self.show_step(3, details=conf['name']) self.create_cluster(conf) self.show_step(4, details=conf['name']) self.update_nodes(conf) self.show_step(5, details=conf['name']) if not self.deploy_cluster(): logger.error("Initial deployment of cluster {0} " "with config name {1} was failed. " "Go to the next config".format( self.cluster_name, conf['name'])) fail_trigger = True failed_confs.append(conf['name']) continue self.show_step(6, details=conf['name']) if not self.run_ostf(): fail_trigger = True failed_confs.append(conf['name']) logger.error( "Failed to pass OSTF tests for first time deployed " "cluster with config {}".format(conf['name'])) continue self.show_step(7, details=conf['name']) attrs = self.get_cluster_attributes() self.show_step(8, details=conf['name']) changer = SettingsChanger(attrs) logger.info("The options below will NOT be changed: {}".format( changer.SKIPPED_FIELDS_LIST)) changer.make_changes(options=None, randomize=30) new_attrs = changer.attrs self.show_step(9, details=conf['name']) ceph_nodes_count = self.check_config_for_ceph(new_attrs) existed_ceph_count = self.get_existed_ceph_nodes_count(conf) if ceph_nodes_count > existed_ceph_count: count = len(conf['nodes']) if count + ceph_nodes_count > settings.NODES_COUNT - 1: logger.info("There are not enough nodes to redeploy with " "ceph nodes pool size. Go to the next config") continue self.add_ceph_nodes(count, ceph_nodes_count) self.show_step(10, details=conf['name']) if not self.update_cluster_attributes(new_attrs): fail_trigger = True failed_confs.append(conf['name']) logger.error( "Failed to update cluster attributes with changed one") continue self.show_step(11, details=conf['name']) if not self.deploy_cluster(): logger.error("Redeployment of cluster {0} " "with config name {1} was failed. " "Go to the next config".format( self.cluster_name, conf['name'])) fail_trigger = True failed_confs.append(conf['name']) continue # Run ostf self.show_step(12, details=conf['name']) if not self.run_ostf(): fail_trigger = True failed_confs.append(conf['name']) logger.error("Failed to pass OSTF tests for redeployed " "cluster with config {}".format(conf['name'])) continue logger.info("Redeployment and OSTF were successfully " "executed for cluster {}".format(self.cluster_name)) self.show_step(13, details=conf['name']) if fail_trigger: assert_false( fail_trigger, "A few configurations were failed: {} " "Please, check logs".format(failed_confs))
def add_custom_nodegroup(self): """Add new nodegroup to operational environment Scenario: 1. Revert snapshot with operational cluster 2. Create new nodegroup for the environment and configure it's networks 3. Bootstrap slave node from custom-2 nodegroup 4. Add node from new nodegroup to the environment with compute role 5. Run network verification 6. Deploy changes 7. Run network verification 8. Run OSTF 9. Check that nodes from 'default' nodegroup can reach nodes from new nodegroup via management and storage networks Duration 50m Snapshot add_custom_nodegroup """ self.show_step(1, initialize=True) self.env.revert_snapshot('deploy_neutron_tun_ha_nodegroups') cluster_id = self.fuel_web.get_last_created_cluster() self.fuel_web.assert_nodes_in_ready_state(cluster_id) asserts.assert_true( not any(ng['name'] == NODEGROUPS[2]['name'] for ng in self.fuel_web.client.get_nodegroups()), 'Custom nodegroup {0} already ' 'exists!'.format(NODEGROUPS[2]['name'])) self.show_step(2) new_nodegroup = self.fuel_web.client.create_nodegroup( cluster_id, NODEGROUPS[2]['name']) logger.debug('Updating custom nodegroup ID in network configuration..') network_config_new = self.fuel_web.client.get_networks(cluster_id) asserts.assert_true(self.netconf_all_groups is not None, 'Network configuration for nodegroups is empty!') for network in self.netconf_all_groups['networks']: if network['group_id'] is not None and \ not any(network['group_id'] == ng['id'] for ng in self.fuel_web.client.get_nodegroups()): network['group_id'] = new_nodegroup['id'] for new_network in network_config_new['networks']: if new_network['name'] == network['name'] and \ new_network['group_id'] == network['group_id']: network['id'] = new_network['id'] self.fuel_web.client.update_network( cluster_id, self.netconf_all_groups['networking_parameters'], self.netconf_all_groups['networks']) self.show_step(3) self.env.bootstrap_nodes([self.env.d_env.nodes().slaves[6]]) self.show_step(4) self.fuel_web.update_nodes( cluster_id, {'slave-07': [['compute'], new_nodegroup['name']]}, True, False) self.show_step(5) self.fuel_web.verify_network(cluster_id) self.show_step(6) self.fuel_web.deploy_cluster_wait(cluster_id) self.show_step(7) self.fuel_web.verify_network(cluster_id) self.show_step(8) self.fuel_web.run_ostf(cluster_id=cluster_id) self.show_step(9) primary_ctrl = self.fuel_web.get_nailgun_node_by_devops_node( self.fuel_web.get_nailgun_primary_node( slave=self.env.d_env.nodes().slaves[0])) with self.fuel_web.get_ssh_for_node('slave-07') as remote: new_node_networks = utils.get_net_settings(remote) for interface in ('br-storage', 'br-mgmt'): if interface in new_node_networks: logger.info("Checking new node is accessible from primary " "controller via {0} interface.".format(interface)) for ip in new_node_networks[interface]['ip_addresses']: address = ip.split('/')[0] result = check_ping(primary_ctrl['ip'], address, timeout=3) asserts.assert_true( result, "New node isn't accessible from " "primary controller via {0} interface" ": {1}.".format(interface, result)) self.env.make_snapshot("add_custom_nodegroup")
def delete_cluster_with_custom_nodegroup(self): """Delete env, check nodes from custom nodegroup can't bootstrap Scenario: 1. Revert snapshot with cluster with nodes in custom nodegroup 2. Delete cluster 3. Check nodes from custom nodegroup can't bootstrap 4. Reset nodes from custom nodegroup 5. Check nodes from custom nodegroup can't bootstrap Duration 15m """ self.show_step(1, initialize=True) self.env.revert_snapshot('deploy_controllers_from_custom_nodegroup') cluster_id = self.fuel_web.get_last_created_cluster() self.fuel_web.assert_nodes_in_ready_state(cluster_id) self.show_step(2) custom_nodes = self.env.d_env.nodes().slaves[3:6] self.fuel_web.delete_env_wait(cluster_id) self.show_step(3) logger.info('Wait five nodes online for 900 seconds..') wait(lambda: len(self.fuel_web.client.list_nodes()) == 5, timeout=15 * 60, timeout_msg='Timeout while waiting five nodes ' 'to become online') logger.info('Wait all nodes from custom nodegroup become ' 'in error state..') # check all custom in error state for slave in custom_nodes: wait(lambda: self.fuel_web.get_nailgun_node_by_devops_node(slave)[ 'status'] == 'error', timeout=15 * 60, timeout_msg='Node {} not changed state to ' 'error'.format(slave.name)) logger.info('Node {} changed state to error'.format(slave.name)) self.show_step(4) logger.info('Rebooting nodes from custom nodegroup..') self.fuel_web.cold_restart_nodes(custom_nodes, wait_online=False) self.show_step(5) logger.info('Wait custom nodes are not online for 600 seconds..') try: wait(lambda: any( self.fuel_web.get_nailgun_node_by_devops_node(slave)['online'] for slave in custom_nodes), timeout=10 * 60) raise AssertionError('Some nodes online') except TimeoutError: logger.info('Nodes are offline') self.env.make_snapshot("delete_cluster_with_custom_nodegroup")
def basic_env_for_numa_cpu_pinning(self): """Basic environment for NUMA CPU pinning Scenario: 1. Create cluster 2. Add 2 nodes with compute role 3. Add 3 nodes with controller role 4. Verify that quantity of NUMA is equal on node and in Fuel Snapshot: basic_env_for_numa_cpu_pinning """ snapshot_name = 'basic_env_for_numa_cpu_pinning' self.check_run(snapshot_name) self.env.revert_snapshot("ready_with_5_slaves") # TODO(kdemina) Use commomn function for variables asserts if not settings.KVM_USE: raise exceptions.FuelQAVariableNotSet( 'KVM_USE', 'true') if int(settings.HARDWARE['slave_node_cpu']) < 6: raise exceptions.FuelQAVariableNotSet( 'SLAVE_NODE_CPU', 6) if int(settings.HARDWARE['numa_nodes']) < 2: raise exceptions.FuelQAVariableNotSet( 'NUMA_NODES', 2) if not settings.INTERFACES_DICT['eth0'] == 'ens3': raise exceptions.FuelQAVariableNotSet( 'IFACE_0', 'ens3') if not settings.INTERFACES_DICT['eth1'] == 'ens4': raise exceptions.FuelQAVariableNotSet( 'IFACE_1', 'ens4') if not settings.INTERFACES_DICT['eth2'] == 'ens5': raise exceptions.FuelQAVariableNotSet( 'IFACE_2', 'ens5') elif not settings.INTERFACES_DICT['eth3'] == 'ens6': raise exceptions.FuelQAVariableNotSet( 'IFACE_3', 'ens6') elif not settings.INTERFACES_DICT['eth4'] == 'ens7': raise exceptions.FuelQAVariableNotSet( 'IFACE_4', 'ens7') elif not settings.INTERFACES_DICT['eth5'] == 'ens8': raise exceptions.FuelQAVariableNotSet( 'IFACE_5', 'ens8') elif not settings.ACPI_ENABLE: raise exceptions.FuelQAVariableNotSet( 'DRIVER_ENABLE_ACPI', 'true') self.show_step(1) cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=settings.DEPLOYMENT_MODE, settings={ "net_provider": 'neutron', "net_segment_type": settings.NEUTRON_SEGMENT_TYPE } ) self.show_step(2) self.show_step(3) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['compute'], 'slave-02': ['compute'], 'slave-03': ['controller'], 'slave-04': ['controller'], 'slave-05': ['controller'] }) self.show_step(4) for node in ('slave-01', 'slave-02'): target_node = self.fuel_web.get_nailgun_node_by_name(node) numas_from_fuel = len( target_node['meta']['numa_topology']['numa_nodes']) numas_on_remote = utils.get_quantity_of_numa(target_node['ip']) if not numas_on_remote: # Fuel handle topology without NUMA as 1 NUMA node asserts.assert_equal(numas_from_fuel, 1, "No NUMA nodes on {0} " "while Fuel shows it " "has {1}".format( target_node['ip'], numas_from_fuel)) raise AssertionError("No NUMA nodes on {0}".format( target_node['ip'])) else: asserts.assert_equal(numas_on_remote, numas_from_fuel, "{0} NUMA nodes on {1} " "while Fuel shows it " "has {2}".format( numas_on_remote, target_node['ip'], numas_from_fuel)) logger.info("There is {0} NUMA nodes on node {1}".format( numas_on_remote, target_node['ip'])) self.env.make_snapshot(snapshot_name, is_make=True)
def deploy_ceph_ha_nodegroups(self): """Deploy HA environment with Neutron VXLAN, Ceph and 2 nodegroups Scenario: 1. Revert snapshot with ready master node 2. Create cluster with Neutron VXLAN, Ceph and custom nodegroup 3. Exclude 10 first IPs from range for default admin/pxe network 4. Bootstrap slave nodes from both default and custom nodegroups 5. Check that excluded IPs aren't allocated to discovered nodes 6. Add 3 controller + ceph nodes from default nodegroup 7. Add 2 compute + ceph nodes from custom nodegroup 8. Deploy cluster 9. Run network verification 10. Run health checks (OSTF) 11. Check that excluded IPs aren't allocated to deployed nodes 12. Check Ceph health Duration 110m Snapshot deploy_ceph_ha_nodegroups """ if not MULTIPLE_NETWORKS: raise SkipTest('MULTIPLE_NETWORKS not enabled') self.show_step(1, initialize=True) self.env.revert_snapshot("ready") self.show_step(2) cluster_id = self.fuel_web.create_cluster(name=self.__class__.__name__, mode=DEPLOYMENT_MODE_HA, settings={ 'volumes_ceph': True, 'images_ceph': True, 'ephemeral_ceph': True, 'volumes_lvm': False, "net_provider": 'neutron', "net_segment_type": NEUTRON_SEGMENT['tun'], 'tenant': 'haVxlanCeph', 'user': '******', 'password': '******' }) self.show_step(3) networks = self.fuel_web.client.get_networks(cluster_id)["networks"] new_admin_range = self.change_default_admin_range( networks, number_excluded_ips=10) wait(lambda: not self.is_update_dnsmasq_running(self.fuel_web.client. get_tasks()), timeout=60, timeout_msg="Timeout exceeded while waiting for task " "'update_dnsmasq' is finished!") self.fuel_web.client.update_network(cluster_id, networks=networks) logger.info("New addresses range for default admin network:" " {0}".format(new_admin_range)) self.show_step(4) self.env.bootstrap_nodes(self.env.d_env.nodes().slaves[0:5]) self.show_step(5) default_ng_nodes = [ self.fuel_web.get_nailgun_node_by_devops_node(node) for node in self.env.d_env.nodes().slaves[0:3] ] for node in default_ng_nodes: asserts.assert_true( self.is_ip_in_range(node['ip'], *new_admin_range), "Node '{0}' has IP address '{1}' which " "is not from defined IP addresses range:" " {2}!".format(node['fqdn'], node['ip'], new_admin_range)) self.show_step(6) self.show_step(7) nodegroup_default = NODEGROUPS[0]['name'] nodegroup_custom = NODEGROUPS[1]['name'] self.fuel_web.update_nodes( cluster_id, { 'slave-01': [['controller', 'ceph-osd'], nodegroup_default], 'slave-02': [['controller', 'ceph-osd'], nodegroup_default], 'slave-03': [['controller', 'ceph-osd'], nodegroup_default], 'slave-04': [['compute', 'ceph-osd'], nodegroup_custom], 'slave-05': [['compute', 'ceph-osd'], nodegroup_custom], }) self.show_step(8) self.fuel_web.deploy_cluster_wait(cluster_id, timeout=150 * 60) self.show_step(9) self.fuel_web.verify_network(cluster_id) self.show_step(10) self.fuel_web.run_ostf(cluster_id=cluster_id) self.show_step(11) group_id = self.fuel_web.get_nodegroup(cluster_id, name=nodegroup_default)['id'] default_ng_nodes = [ node for node in self.fuel_web.client.list_cluster_nodes(cluster_id) if node['group_id'] == group_id ] for node in default_ng_nodes: asserts.assert_true( self.is_ip_in_range(node['ip'], *new_admin_range), "Node '{0}' has IP address '{1}' which " "is not from defined IP addresses range:" " {2}!".format(node['fqdn'], node['ip'], new_admin_range)) self.show_step(12) self.fuel_web.check_ceph_status(cluster_id) self.env.make_snapshot("deploy_ceph_ha_nodegroups")
def contrail_plugin_add_delete_controller_node(self): """Verify that Controller node can be deleted and added after deploying Scenario: 1. Revert snapshot "ready_with_9_slaves" 2. Create cluster 3. Add 3 nodes with Operating system role, 2 nodes with controller role and 1 node with compute role 4. Enable Contrail plugin 5. Deploy cluster with plugin 6. Remove 1 node with controller role. 7. Deploy cluster 8. Add 1 nodes with controller role 9. Deploy cluster 10. Run OSTF tests Duration 140 min """ self._prepare_contrail_plugin(slaves=9) # create cluster: 3 nodes with Operating system role # and 1 node with controller role self.fuel_web.update_nodes( self.cluster_id, { 'slave-01': ['base-os'], 'slave-02': ['base-os'], 'slave-03': ['base-os'], 'slave-04': ['controller'], 'slave-05': ['controller'], 'slave-06': ['controller'], 'slave-07': ['compute'] }, custom_names={ 'slave-01': 'contrail-1', 'slave-02': 'contrail-2', 'slave-03': 'contrail-3' } ) # configure disks on base-os nodes self.change_disk_size() # enable plugin in contrail settings self._activate_plugin() self.fuel_web.deploy_cluster_wait(self.cluster_id, check_services=False, timeout=240 * 60) # remove one node with controller role self.fuel_web.update_nodes( self.cluster_id, {'slave-05': ['controller']}, False, True) self.fuel_web.deploy_cluster_wait(self.cluster_id, check_services=False, timeout=240 * 60) # add 1 node with controller role and redeploy cluster self.fuel_web.update_nodes( self.cluster_id, {'slave-08': ['controller']}) self.fuel_web.deploy_cluster_wait(self.cluster_id, check_services=False, timeout=240 * 60) # TODO # Tests using north-south connectivity are expected to fail because # they require additional gateway nodes, and specific contrail # settings. This mark is a workaround until it's verified # and tested manually. # Also workaround according to bug 1457515 # When it will be done 'should_fail=3' and # 'failed_test_name' parameter should be removed. # create net and subnet to pass ostf self._create_net_subnet(self.cluster_id) self.fuel_web.run_ostf( cluster_id=self.cluster_id, should_fail=2, failed_test_name=[('Check network connectivity ' 'from instance via floating IP'), 'Launch instance with file injection'] ) logger.info(self._ostf_msg)
def main(): parser = argparse.ArgumentParser(description=""" Example: python helpers/conf_tempest.py -c 1 \ -n 10.108.10.2 \ -t /home/fuel/tempest/etc/tempest.conf """) parser.add_argument("-n", "--nailgun", help="Provide nailgun node ip.", required=True) parser.add_argument("-c", "--cluster", help="Provide cluster id", required=True) parser.add_argument("-t", "--tempest_config", help="Path where tempest will look for config", default='/etc/tempest/tempest.conf') args = parser.parse_args() conf = TempestConfigState(args.nailgun, args.cluster, tempest_conf=args.tempest_config) conf.configure() conf.copy_config() if __name__ == '__main__': logger.info('Starting tempest config generation.') main() logger.info('Finished tempest config generation.')
def ceph_ha_restart(self): """Destructive ceph test in HA mode Scenario: 1. Revert from ceph_ha 2. Waiting up galera and cinder 3. Check ceph status 4. Run OSTF 5. Destroy and remove osd-node 6. Check ceph status 7. Run OSTF 8. Destroy and remove one compute node 9. Check ceph status 10. Run OSTF 11. Cold restart 12. Waiting up galera and cinder 13. Run single OSTF - Create volume and attach it to instance 14. Run OSTF Duration 30m Snapshot ceph_ha_restart """ self.env.revert_snapshot("ceph_ha") # Wait until MySQL Galera is UP on some controller self.fuel_web.wait_mysql_galera_is_up(['slave-01']) # Wait until Cinder services UP on a controller self.fuel_web.wait_cinder_is_up(['slave-01']) cluster_id = self.fuel_web.get_last_created_cluster() self.fuel_web.check_ceph_status(cluster_id) # Run ostf self.fuel_web.run_ostf(cluster_id=cluster_id) # Destroy and remove osd-node logger.info("Destroy and remove slave-06") with self.fuel_web.get_ssh_for_node('slave-06') as remote_ceph: self.fuel_web.prepare_ceph_to_delete(remote_ceph) slave_06 = self.env.d_env.get_node(name='slave-06') nailgun_node_id = self.fuel_web.get_nailgun_node_by_devops_node( slave_06)['id'] slave_06.destroy() wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node( slave_06)['online'], timeout=30 * 8) self.fuel_web.delete_node(nailgun_node_id) self.fuel_web.check_ceph_status(cluster_id) self.fuel_web.run_ostf(cluster_id=cluster_id, test_sets=['sanity', 'smoke', 'ha'], should_fail=1) # Destroy and remove compute node logger.info("Destroy and remove slave-05") with self.fuel_web.get_ssh_for_node('slave-05') as remote_ceph: self.fuel_web.prepare_ceph_to_delete(remote_ceph) slave_05 = self.env.d_env.get_node(name='slave-05') nailgun_node_id = self.fuel_web.get_nailgun_node_by_devops_node( slave_05)['id'] slave_05.destroy() wait(lambda: not self.fuel_web.get_nailgun_node_by_devops_node( slave_05)['online'], timeout=30 * 8) self.fuel_web.delete_node(nailgun_node_id) self.fuel_web.check_ceph_status(cluster_id) self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1) # Cold restart self.fuel_web.cold_restart_nodes( self.env.d_env.get_nodes( name__in=['slave-01', 'slave-02', 'slave-03', 'slave-04'])) # Wait for HA services ready self.fuel_web.assert_ha_services_ready(cluster_id) # Wait until OpenStack services are UP, should fail 2 services # because slave-05 (compute+ceph-osd) destroyed. We ignore # expect fail a test - 'Check openstack services are running' self.fuel_web.assert_os_services_ready(cluster_id, should_fail=1) self.fuel_web.check_ceph_status(cluster_id) # Wait until MySQL Galera is UP on some controller self.fuel_web.wait_mysql_galera_is_up(['slave-01']) # Wait until Cinder services UP on a controller self.fuel_web.wait_cinder_is_up(['slave-01']) try: self.fuel_web.run_single_ostf_test( cluster_id, test_sets=['smoke'], test_name=map_ostf.OSTF_TEST_MAPPING.get( 'Create volume and attach it to instance')) except AssertionError: logger.debug("Test failed from first probe," " we sleep 60 second try one more time " "and if it fails again - test will fails ") time.sleep(180) self.fuel_web.run_single_ostf_test( cluster_id, test_sets=['smoke'], test_name=map_ostf.OSTF_TEST_MAPPING.get( 'Create volume and attach it to instance')) self.fuel_web.run_ostf(cluster_id=cluster_id, should_fail=1) self.env.make_snapshot("ceph_ha_restart")
def _install_packages(self, remote): command = "cd " + self._pack_copy_path + " && ./install.sh" logger.info('The command is %s', command) remote.execute_async(command) time.sleep(50) os.path.isfile(self._add_ub_packag or self._add_cen_packeg)
def deploy_ha_cgroup(self): """Deploy cluster in HA mode with enabled cgroups Scenario: 1. Create cluster 2. Add 3 nodes with controller role 3. Add 1 node with compute role 4. Add 1 node with cinder role 5. Deploy the cluster 6. Check ceph status 7. Run OSTF Duration 90m Snapshot deploy_ha_cgroup """ self.check_run("deploy_ha_cgroup") self.env.revert_snapshot("ready_with_5_slaves") data = { 'tenant': 'cgroup', 'user': '******', 'password': '******', 'net_provider': 'neutron', 'net_segment_type': settings.NEUTRON_SEGMENT['vlan'] } cluster_id = self.fuel_web.create_cluster( name=self.__class__.__name__, mode=settings.DEPLOYMENT_MODE, settings=data) self.fuel_web.update_nodes( cluster_id, { 'slave-01': ['controller'], 'slave-02': ['controller'], 'slave-03': ['controller'], 'slave-04': ['compute'], 'slave-05': ['cinder'] }) cgroup_data = [{ "process": "keystone", "controller": "cpu", "limit": "cpu.shares", "value": 70, }] self.fuel_web.client.update_cluster_attributes( cluster_id, self.generate_attributes(cgroup_data)) # Cluster deploy self.fuel_web.deploy_cluster_wait(cluster_id) # Run ostf self.fuel_web.run_ostf(cluster_id=cluster_id) # Check that task cgroup was executed cmd = 'fgrep "MODULAR: cgroups/cgroups.pp" -q /var/log/puppet.log' n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles( cluster_id, ['controller']) for nailgun_node in n_ctrls: logger.info('Check cgroups task on controller node {0}'.format( nailgun_node["fqdn"])) self.ssh_manager.check_call(nailgun_node['ip'], cmd) self.check_cgroups_on_node(nailgun_node, cgroup_data) self.env.make_snapshot("deploy_ha_cgroup", is_make=True)
def deploy_ha_contrail_plugin(self): """Deploy HA Environment with Contrail Plugin Scenario: 1. Revert snapshot "ready_with_9_slaves" 2. Create cluster 3. Add 3 nodes with Operating system role and 1 node with controller role 4. Enable Contrail plugin 5. Deploy cluster with plugin 6. Add 1 node with compute role 7. Deploy cluster 8. Run OSTF tests 9. Add 2 nodes with controller role and 1 node with compute + cinder role 10. Deploy cluster 11. Run OSTF tests Duration 140 min """ self._prepare_contrail_plugin(slaves=9) # create cluster: 3 nodes with Operating system role # and 1 node with controller role self.fuel_web.update_nodes( self.cluster_id, { 'slave-01': ['base-os'], 'slave-02': ['base-os'], 'slave-03': ['base-os'], 'slave-04': ['controller'] }, custom_names={ 'slave-01': 'contrail-1', 'slave-02': 'contrail-2', 'slave-03': 'contrail-3' } ) # configure disks on base-os nodes self.change_disk_size() # enable plugin in contrail settings self._activate_plugin() self.fuel_web.deploy_cluster_wait(self.cluster_id) # create net and subnet self._create_net_subnet(self.cluster_id) # add 1 node with compute role and redeploy cluster self.fuel_web.update_nodes( self.cluster_id, {'slave-05': ['compute']},) self.fuel_web.deploy_cluster_wait(self.cluster_id) # TODO # Tests using north-south connectivity are expected to fail because # they require additional gateway nodes, and specific contrail # settings. This mark is a workaround until it's verified # and tested manually. # When it will be done 'should_fail=2' and # 'failed_test_name' parameter should be removed. self.fuel_web.run_ostf( cluster_id=self.cluster_id, should_fail=2, failed_test_name=[('Check network connectivity ' 'from instance via floating IP'), 'Launch instance with file injection'] ) logger.info(self._ostf_msg) # add to cluster 2 nodes with controller role and one # with compute, cinder role and deploy cluster self.fuel_web.update_nodes( self.cluster_id, { 'slave-06': ['controller'], 'slave-07': ['controller'], 'slave-08': ['compute', 'cinder'], } ) logger.info(self._ostf_msg) self.fuel_web.deploy_cluster_wait(self.cluster_id) # TODO: # Tests using north-south connectivity are expected to fail because # they require additional gateway nodes, and specific contrail # settings. This mark is a workaround until it's verified # and tested manually. # When it will be done 'should_fail=2' and # 'failed_test_name' parameter should be removed. self.fuel_web.run_ostf( cluster_id=self.cluster_id, should_fail=2, failed_test_name=[('Check network connectivity ' 'from instance via floating IP'), 'Launch instance with file injection'] )
def cli_selected_nodes_deploy(self): """Create and deploy environment using Fuel CLI and check CN name is equal to the public name passed via UI (user-owned cert) Scenario: 1. Create environment using fuel-qa 2. Create a cluster using Fuel CLI 3. Add floating ranges for public network 4. Get cluster settings 5. Provision a controller node using Fuel CLI 6. Provision two compute+cinder nodes using Fuel CLI 7. Deploy the controller node using Fuel CLI 8. Deploy the compute+cinder nodes using Fuel CLI 9. Compare network settings after compute deployment task 10. Verify network 11. Check that all services work by 'https' 12. Check that all services have domain name 13. Find 'CN' value at the output: CN value is equal to the value specified at certificate provided via Fuel UI 14. Find keypair data at the output: Keypair data is equal to the value specified at certificate provided via Fuel UI 15. Compare floating ranges 16. Get deployment-info 17. Get cluster settings after deployment task 18. Compare cluster settings after deploy and before deploy 19. Run OSTF Duration 50m """ self.env.revert_snapshot("ready_with_3_slaves") node_ids = sorted( [node['id'] for node in self.fuel_web.client.list_nodes()]) release_id = self.fuel_web.get_releases_list_for_os( release_name=OPENSTACK_RELEASE)[0] admin_ip = self.ssh_manager.admin_ip # Create an environment self.show_step(1) if NEUTRON_SEGMENT_TYPE: nst = '--nst={0}'.format(NEUTRON_SEGMENT_TYPE) else: nst = '' self.show_step(2) cmd = ('fuel env create --name={0} --release={1} {2} --json'.format( self.__class__.__name__, release_id, nst)) env_result =\ self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] cluster_id = env_result['id'] self.show_step(3) # Update network parameters self.update_cli_network_configuration(cluster_id) # Change floating ranges current_floating_range = self.get_floating_ranges(cluster_id) logger.info( "Current floating ranges: {0}".format(current_floating_range)) first_floating_address = current_floating_range[0][0] logger.info( "First floating address: {0}".format(first_floating_address)) last_floating_address = current_floating_range[0][1] logger.info("Last floating address: {0}".format(last_floating_address)) new_floating_range = generate_floating_ranges(first_floating_address, last_floating_address, 10) logger.info("New floating range: {0}".format(new_floating_range)) self.change_floating_ranges(cluster_id, new_floating_range) # Update SSL configuration self.update_ssl_configuration(cluster_id) # Get cluster settings before deploy self.show_step(4) cluster_settings = self.download_settings(cluster_id) self.show_step(5) # Add and provision a controller node logger.info("Add to the cluster \ and start provisioning a controller node [{0}]".format(node_ids[0])) cmd = ('fuel --env-id={0} node set --node {1}\ --role=controller'.format(cluster_id, node_ids[0])) self.ssh_manager.execute_on_remote(admin_ip, cmd) self.update_node_interfaces(node_ids[0]) cmd = ('fuel --env-id={0} node --provision --node={1} --json'.format( cluster_id, node_ids[0])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] self.assert_cli_task_success(task, timeout=30 * 60) self.show_step(6) # Add and provision 2 compute+cinder logger.info("Add to the cluster and start provisioning two " "compute+cinder nodes [{0},{1}]".format( node_ids[1], node_ids[2])) cmd = ('fuel --env-id={0} node set --node {1},{2} \ --role=compute,cinder'.format(cluster_id, node_ids[1], node_ids[2])) self.ssh_manager.execute_on_remote(admin_ip, cmd) for node_id in (node_ids[1], node_ids[2]): self.update_node_interfaces(node_id) cmd = ('fuel --env-id={0} node --provision \ --node={1},{2} --json'.format(cluster_id, node_ids[1], node_ids[2])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] self.assert_cli_task_success(task, timeout=10 * 60) self.show_step(7) # Deploy the controller node cmd = ('fuel --env-id={0} node --deploy --node {1} --json'.format( cluster_id, node_ids[0])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] self.assert_cli_task_success(task, timeout=60 * 60) self.assert_all_tasks_completed(cluster_id=cluster_id) self.show_step(8) # Deploy the compute nodes cmd = ('fuel --env-id={0} node --deploy --node {1},{2} --json'.format( cluster_id, node_ids[1], node_ids[2])) task = self.ssh_manager.execute_on_remote(admin_ip, cmd, jsonify=True)['stdout_json'] self.wait_cli_task_status(task=task, status='running') # Fuel 9.1 is async, so we should wait for real task start network_settings = self.get_networks(cluster_id) self.assert_cli_task_success(task, timeout=30 * 60) self.assert_all_tasks_completed(cluster_id=cluster_id) # Verify networks self.show_step(9) network_configuration = self.get_net_config_cli() assert_equal(network_settings, network_configuration, message='Network settings are not equal before' ' and after deploy') self.show_step(10) self.fuel_web.verify_network(cluster_id) controller_nodes = self.fuel_web.get_nailgun_cluster_nodes_by_roles( cluster_id, ['controller']) # Get controller ip address controller_node = controller_nodes[0]['ip'] # Get endpoint list endpoint_list = self.get_endpoints(controller_node) logger.info(endpoint_list) # Check protocol and domain names for endpoints self.show_step(11) self.show_step(12) for endpoint in endpoint_list: logger.debug(("Endpoint {0} use protocol {1}\ and have domain name {2}".format(endpoint['service_name'], endpoint['protocol'], endpoint['domain']))) assert_equal(endpoint['protocol'], "https", message=("Endpoint {0} don't use https.".format( endpoint['service_name']))) assert_equal(endpoint['domain'], SSL_CN, message=("{0} domain name not equal {1}.".format( endpoint['service_name'], SSL_CN))) self.show_step(13) current_ssl_cn = self.get_current_ssl_cn(controller_node) logger.info(("CN before cluster deploy {0} \ and after deploy {1}".format(SSL_CN, current_ssl_cn))) assert_equal(SSL_CN, current_ssl_cn, message="SSL CNs are not equal") self.show_step(14) with open(PATH_TO_PEM) as pem_file: old_ssl_keypair = pem_file.read().strip() current_ssl_keypair = self.get_current_ssl_keypair(controller_node) logger.info("SSL keypair before cluster deploy:\n" "{0}\n" "and after deploy:\n" "{1}".format(old_ssl_keypair, current_ssl_keypair)) assert_equal(old_ssl_keypair, current_ssl_keypair, message="SSL keypairs are not equal") self.show_step(15) actual_floating_ranges = self.hiera_floating_ranges(controller_node) logger.info( "Current floating ranges: {0}".format(actual_floating_ranges)) assert_equal(actual_floating_ranges, new_floating_range, message="Floating ranges are not equal") # Get deployment task id task_id = self.get_first_task_id_by_name(cluster_id, 'deployment') self.show_step(16) # Get deployment info self.get_deployment_info_cli(task_id) self.show_step(17) # Get cluster settings after deploy cluster_config = self.get_cluster_config_cli(task_id) self.show_step(18) # Compare cluster settings assert_equal(cluster_settings, cluster_config, message='Cluster settings are not equal before' ' and after deploy') self.show_step(19) # Run OSTF self.fuel_web.run_ostf(cluster_id=cluster_id, test_sets=['ha', 'smoke', 'sanity']) self.env.make_snapshot("cli_selected_nodes_deploy", is_make=True)
def apply_relative_cgroups_after_deploy(self): """Apply relative cgroups limits to services Scenario: 1. Revert snapshot deploy_ha_cgroup 2. Configure and validate cgroups for mysqld, rabbitmq and keystone with relative memory count Duration 15m """ self.show_step(1) self.env.revert_snapshot("deploy_ha_cgroup") cluster_id = self.fuel_web.get_last_created_cluster() n_ctrls = self.fuel_web.get_nailgun_cluster_nodes_by_roles( cluster_id, ['controller']) ctrl_ids = ",".join( [str(nailgun_node['id']) for nailgun_node in n_ctrls]) self.show_step(2) cgroups = [ { "process": "mysqld", "controller": "memory", "limit": "memory.swappiness", "value": 0 }, { "process": "mysqld", "controller": "memory", "limit": "memory.soft_limit_in_bytes", "value": "%5,10,3000" }, { "process": "rabbitmq", "controller": "blkio", "limit": "blkio.weight", "value": 500 }, { "process": "rabbitmq", "controller": "memory", "limit": "memory.soft_limit_in_bytes", "value": "%99,10,250" }, { "process": "keystone", "controller": "cpu", "limit": "cpu.shares", "value": 50 }, { "process": "keystone", "controller": "memory", "limit": "memory.soft_limit_in_bytes", "value": "%1,250,2500" }, ] self.apply_cgroups(cgroups, ctrl_ids) memory = float("".join( self.ssh_manager.execute(n_ctrls[0]["ip"], "facter memorysize_mb")["stdout"])) for cgroup in cgroups: if cgroup["limit"] == "memory.soft_limit_in_bytes": # pylint: disable=no-member percent, min_mem, max_mem = cgroup["value"].split(",") # pylint: enable=no-member percent = int(percent.replace("%", "")) * memory / 100 min_mem, max_mem = int(min_mem), int(max_mem) value = sorted((min_mem, percent, max_mem))[1] cgroup["value"] = int(value * 1024 * 1024) logger.info("New cgroups to verify: {}".format(cgroups)) for nailgun_node in n_ctrls: self.check_cgroups_on_node(nailgun_node, cgroups)
def replace_fuel_nailgun_rpm(environment): """ Replace fuel_nailgun*.rpm from review environment - Environment Model object - self.env """ logger.info("Patching fuel-nailgun") if not settings.UPDATE_FUEL: raise exceptions.FuelQAVariableNotSet('UPDATE_FUEL', 'True') pack_path = '/var/www/nailgun/fuel-nailgun/' full_pack_path = os.path.join(pack_path, 'fuel-nailgun*.noarch.rpm') logger.info('Package path {0}'.format(full_pack_path)) with environment.d_env.get_admin_remote() as remote: remote.upload(settings.UPDATE_FUEL_PATH.rstrip('/'), pack_path) # Check old fuel-nailgun package cmd = "rpm -q fuel-nailgun" old_package = environment.base_actions.execute(cmd, exit_code=0) logger.info('Current package version of ' 'fuel-nailgun: {0}'.format(old_package)) cmd = "rpm -qp {0}".format(full_pack_path) new_package = environment.base_actions.execute(cmd) logger.info("Updating package {0} with {1}".format(old_package, new_package)) if old_package == new_package: logger.debug('Looks like package from review ' 'was installed during setups of master node') return # stop services service_list = ['assassind', 'receiverd', 'nailgun', 'statsenderd'] [ environment.base_actions.execute('systemctl stop {0}'.format(service), exit_code=0) for service in service_list ] # stop statistic services [ environment.base_actions.execute('systemctl stop {0}'.format(service), exit_code=0) for service in get_oswl_services_names(environment) ] # Drop nailgun db manage.py dropdb cmd = 'manage.py dropdb' environment.base_actions.execute(cmd, exit_code=0) # Delete package logger.info("Delete package {0}".format(old_package)) cmd = "rpm -e fuel-nailgun" environment.base_actions.execute(cmd, exit_code=0) logger.info("Install package {0}".format(new_package)) cmd = "rpm -Uvh --oldpackage {0}".format(full_pack_path) environment.base_actions.execute(cmd, exit_code=0) cmd = "rpm -q fuel-nailgun" installed_package = environment.base_actions.execute(cmd, exit_code=0) assert_equal(installed_package, new_package, "The new package {0} was not installed".format(new_package)) cmd = ('puppet apply --debug ' '/etc/puppet/modules/fuel/examples/nailgun.pp') environment.base_actions.execute(cmd, exit_code=0) with environment.d_env.get_admin_remote() as remote: res = remote.execute( "fuel release --sync-deployment-tasks --dir /etc/puppet/") assert_equal(res['exit_code'], 0, 'Failed to sync tasks with result {0}'.format(res))
def get_defaults(): """Get default parameters from config.yaml.""" with open('plugin_test/helpers/config.yaml') as config: defaults = yaml.load(config.read()) logger.info(''.format(defaults)) return defaults