def _get_dpdk_nics_info(self, hypervisor_ip): dpdk_nics_info = [] dpdk_nics = [] cmd = ("sudo ovs-vsctl --columns=name,type,admin_state " "--format=json list interface") output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) nics = json.loads(output) for nic in nics.get('data', []): if nic and str(nic[1]) == 'dpdk' and str(nic[2]) == 'up': dpdk_nics.append(str(nic[0])) if dpdk_nics: cmd = ("sudo ovs-vsctl --column=mac-in-use,mtu,status " "--format=json list interface " + ' '.join(dpdk_nics)) output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) nics_info = json.loads(output) for nic_info in nics_info.get('data', []): data = {} data['mac'] = nic_info[0] data['mtu'] = nic_info[1] for field in nic_info[2][1]: if field[0] == 'numa_id': data['numa_node'] = int(field[1]) dpdk_nic_map = self._get_dpdk_nics_mapping( hypervisor_ip, nic_info[0]) data['nic'] = dpdk_nic_map['name'] data['pci'] = dpdk_nic_map['pci_address'] dpdk_nics_info.append(data) return dpdk_nics_info
def test_offload_nic_eswitch_mode(self): """Check eswitch mode of nic for offload on all hypervisors By default, offload nics are auto discovered. But if the used would like to not perform the autodiscover and provide the nics, it could be done by modifying the CONF.nfv_plugin_options.offload_nics param in deployer-input file. """ LOG.info('Starting offload_nic_eswitch_mode test') # Retrieve all hypervisors hypervisors = self._get_hypervisor_ip_from_undercloud() offload_nics = CONF.nfv_plugin_options.offload_nics if not offload_nics: LOG.info('The offload nics are not provided. Detecting...') offload_nics = self.discover_hw_offload_nics(hypervisors) LOG.info('Test the following offload nics - {}'.format(offload_nics)) # devlink cmd to retrieve switch mode of interface devlink_cmd = "sudo devlink dev eswitch show pci/{}" # Initialize results list result = [] # Expected result is a list of dicts containing a dict of # hypervisor's IP, its offload nics as keys and the value 'true' # Example: # [{'192.0.160.1': [{'p6p1': 'true'}, {'p6p2': 'true'}]}, # {'192.0.160.2': [{'p6p1': 'true'}, {'p6p2': 'true'}]}] expected_result = [{ip: [{nic: 'true'} for nic, _ in nics.items()]} for ip, nics in offload_nics.items()] for hypervisor, nics in offload_nics.items(): dev_result = [] # Check hw-offload config on hypervisor hyper_check = \ 'sudo ovs-vsctl get Open_vSwitch . other_config:hw-offload' hyper_offload_state = shell_utils.run_command_over_ssh(hypervisor, hyper_check) if not hyper_offload_state.strip() == '"true"': dev_result.append('No hw-offload on hypervisor') result.append({hypervisor: dev_result}) LOG.info('No hw-offload on hypervisor {}'.format(hypervisor)) continue LOG.info('Hw-offload configured on hyper - {}'.format(hypervisor)) for nic, nic_options in nics.items(): dev_query = shell_utils.run_command_over_ssh( hypervisor, devlink_cmd.format(nic_options['bus-info'])) if 'switchdev' in dev_query: output = 'true' else: output = 'false' LOG.info("Hypervisor '{h}' NIC '{n}' is in switchdev mode: {r}" .format(h=hypervisor, n=nic, r=output)) dev_result.append({nic: output}) result.append({hypervisor: dev_result}) msg = "Not all hypervisors contains nics in switchev mode" self.assertItemsEqual(expected_result, result, msg)
def test_hypervisor_reboot(self, test='hypervisor_reboot'): """Test functionality of DPDK and SRIOV after hypervisor reboot The test will spawn up an instance and then will reboot the hypervisor that holds the test instance. After hypervisor boot up, the instance will be started and tested for the accessability. """ servers, key_pair = self.create_and_verify_resources(test=test) # Ensure that we are using microversion '2.32' from now self.useFixture( api_microversion_fixture.APIMicroversionFixture('2.32')) LOG.info("Locate instance hypervisor") srv_hyper_name = self.os_admin.servers_client.show_server( servers[0]['id'])['server']['OS-EXT-SRV-ATTR:host'] srv_on_hyper = self.hypervisor_client.list_servers_on_hypervisor( srv_hyper_name)['hypervisors'][0]['servers'] LOG.info("Shut down the instances and reboot the hypervisor " "the instance resides on") # In order the prevent instances file system corruption, # shut down the instance. for srv in srv_on_hyper: self.servers_client.stop_server(srv['uuid']) waiters.wait_for_server_status(self.servers_client, srv['uuid'], 'SHUTOFF') shell_utils.run_command_over_ssh(servers[0]['hypervisor_ip'], "sudo reboot") # Reboot of the baremetal hypervisor takes time. # In order to not confuse the test, look for the hypervisor status # "down" and then "up". hyper_rebooted = False timeout_start = time.time() timeout_end = CONF.nfv_plugin_options.hypervisor_wait_timeout while time.time() < timeout_start + timeout_end: time.sleep(10) hyper_state = self.hypervisor_client.search_hypervisor( srv_hyper_name)['hypervisors'][0]['state'] if 'down' in hyper_state: hyper_rebooted = True continue if hyper_rebooted and 'up' in hyper_state: break LOG.info("Hypervisor has been rebooted. Booting up the instances.") for srv in srv_on_hyper: self.servers_client.start_server(srv['uuid']) waiters.wait_for_server_status(self.servers_client, srv['uuid'], 'ACTIVE') LOG.info("Check instances connectivity") for srv in servers: self.check_instance_connectivity(ip_addr=srv['fip'], user=self.instance_user, key_pair=key_pair['private_key']) LOG.info("The hypervisor reboot test passed.")
def test_restart_ovs(self, test='restart_ovs'): """Test restart_ovs Check that config is loaded properly """ LOG.info('Starting restart_ovs test.') hypervisor_ip = self._get_hypervisor_ip_from_undercloud( shell='/home/stack/stackrc')[0] cmd = 'sudo systemctl restart openvswitch.service' shell_utils.run_command_over_ssh(hypervisor_ip, cmd) self.test_deployment_lacp(hypervisor_ip=hypervisor_ip)
def test_igmp_restart_ovs(self, test='igmp_restart_ovs'): """Test restart ovs Check that multicast configuration is not lost after ovs restart. Restart ovs and then execute test_igmp_snooping_deployment """ LOG.info('Starting {} test.'.format(test)) hypervisor_ips = self._get_hypervisor_ip_from_undercloud( shell='/home/stack/stackrc') cmd = 'sudo systemctl restart openvswitch.service' for hyp in hypervisor_ips: shell_utils.run_command_over_ssh(hyp, cmd) self.test_igmp_snooping_deployment()
def test_offload_ovs_config(self): """Check ovs config for offload on all hypervisors """ # Command to check if hw-offload is enabled in OVS cmd = ("sudo ovs-vsctl get open_vswitch . " "other_config:hw-offload") # Retrieve all hypvervisors hypervisors = self._get_hypervisor_ip_from_undercloud( shell='/home/stack/stackrc') # Intialize results list result = [] # Expected result is a list of dicts, each dict contains # a key which is hypervisor's IP and the value 'true' # Example: # [{192.0.60.1: 'true'}, {192.0.60.2: 'true'}] expected_result = [{ip: 'true'} for ip in hypervisors] for hypervisor in hypervisors: out = shell_utils.run_command_over_ssh(hypervisor, cmd) if out: # Strip newlines and remove double quotes output = out.rstrip().replace('"', '') # HW-Offload not enabled if no text returned else: output = 'false' LOG.info("Hypervisor '{h}' is OVS HW-offload " "capable: '{r}'".format(h=hypervisor, r=output)) result.append({hypervisor: output}) msg = "Not all hypervisors have OVS HW-Offload enabled" self.assertItemsEqual(expected_result, result, msg)
def test_restart_ovs(self, test='restart_ovs'): """Test restart_ovs Check that config is loaded properly """ LOG.info('Starting restart_ovs test.') hypervisor_ip = self._get_hypervisor_ip_from_undercloud()[0] cmd = 'sudo systemctl restart openvswitch.service' shell_utils.run_command_over_ssh(hypervisor_ip, cmd) self.test_deployment_lacp(hypervisor_ip=hypervisor_ip) # Give time to have everything up after reboot so that other testcases # executed after this one do not fail time.sleep(60)
def test_deployment_lacp(self, test='deployment_lacp', hypervisor_ip=None): """Check that lacp bonding is properly configure Configuration options example: - name: deployment_lacp bonding_config: - bond_name: 'dpdkbond1' bond_mode: 'balance-tcp' lacp_status: 'negotiated' lacp_time: 'fast' lacp_fallback_ab: 'true' """ LOG.info('Starting deployment_lacp test.') if hypervisor_ip is None: hypervisor_ip = self._get_hypervisor_ip_from_undercloud( shell='/home/stack/stackrc')[0] bonding_dict = {} test_setup_dict = self.test_setup_dict[test] if 'config_dict' in test_setup_dict and \ 'bonding_config' in test_setup_dict['config_dict']: bonding_dict = test_setup_dict['config_dict']['bonding_config'][0] cmd = 'sudo ovs-appctl bond/show {0} | '\ 'egrep "^bond_mode|^lacp_status|^lacp_fallback_ab"; '\ 'sudo ovs-appctl lacp/show {0} | '\ 'egrep "lacp_time"'.format(bonding_dict['bond_name']) output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) \ .replace('\t', '').replace(' ', '').split('\n') bond_data = {} for i in range(len(output)): data = output[i].split(':') if len(data) == 2: bond_data[data[0]] = data[1] result = [] checks = {'bond_mode', 'lacp_status', 'lacp_time', 'lacp_fallback_ab'} diff_checks_cmd = checks - set(bond_data.keys()) diff_checks_cfg = checks - set(bonding_dict.keys()) if len(diff_checks_cmd) > 0: result.append("Missing checks: {}. Check ovs commands " "output".format(', '.join(diff_checks_cmd))) if len(diff_checks_cmd) > 0: result.append("Missing checks: {}. Check testcase config " "file".format(', '.join(diff_checks_cfg))) for check in checks: if check not in diff_checks_cmd and \ check not in diff_checks_cfg: if bond_data[check] != bonding_dict[check]: result.append("Check failed: {}, Expected: {} - " "Found: {}".format(check, bonding_dict[check], bond_data[check])) self.assertTrue(len(result) == 0, '. '.join(result)) return True
def _get_numa_nodes(self, hypervisor_ip): nodes = [] cmd = "sudo lscpu -p=NODE | grep -v ^#" output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) for line in output.split('\n'): if line: node = int(line.strip(' ')) if node not in nodes: nodes.append(node) return nodes
def test_deployment_lacp(self, hypervisor_ip=None): """Check that lacp bonding is properly configured The test uses the following configuration options example: bond_mode: 'balance-tcp' lacp_status: 'negotiated' lacp_time: 'fast' lacp_fallback_ab: 'true' The "bond_name" is auto discovered. """ LOG.info('Starting deployment_lacp test.') if hypervisor_ip is None: hypervisor_ip = self._get_hypervisor_ip_from_undercloud()[0] lacp_config = json.loads(CONF.nfv_plugin_options.lacp_config) lacp_bond = self.retrieve_lacp_ovs_bond(hypervisor_ip) cmd = 'sudo ovs-appctl bond/show {0} | '\ 'egrep "^bond_mode|^lacp_status|^lacp_fallback_ab"; '\ 'sudo ovs-appctl lacp/show {0} | '\ 'egrep "lacp_time"'.format(lacp_bond['bond_name']) output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) \ .replace('\t', '').replace(' ', '').split('\n') bond_data = {} for i in range(len(output)): data = output[i].split(':') if len(data) == 2: bond_data[data[0]] = data[1] result = [] checks = {'bond_mode', 'lacp_status', 'lacp_time', 'lacp_fallback_ab'} diff_checks_cmd = checks - set(bond_data.keys()) diff_checks_cfg = checks - set(lacp_config.keys()) if len(diff_checks_cmd) > 0: result.append("Missing checks: {}. Check ovs commands " "output".format(', '.join(diff_checks_cmd))) if len(diff_checks_cmd) > 0: result.append("Missing checks: {}. Check testcase config " "file".format(', '.join(diff_checks_cfg))) for check in checks: if check not in diff_checks_cmd and \ check not in diff_checks_cfg: if bond_data[check] != lacp_config[check]: result.append("Check failed: {}, Expected: {} - " "Found: {}".format(check, lacp_config[check], bond_data[check])) self.assertTrue(len(result) == 0, '. '.join(result)) return True
def test_offload_ovs_flows(self, test='offload_flows'): """Check OVS offloaded flows The following test deploy vms, on hw-offload computes. It sends async ping and check offload flows exist in ovs. :param test: Test name from the external config file. """ LOG.info('Start test_offload_ovs_flows test.') LOG.info('test_offload_ovs_flows create vms') # Create servers servers, key_pair = self.create_and_verify_resources(test=test, num_servers=4) cmd = 'sudo ovs-appctl dpctl/dump-flows type=offloaded' # Iterate over created servers for server in servers: shell_utils.continuous_ping(server['fip'], duration=30) LOG.info('test_offload_ovs_flows verify flows on geust {}'. format(server['fip'])) out = shell_utils.\ run_command_over_ssh(server['hypervisor_ip'], cmd) ports = \ self.os_admin.ports_client.list_ports(device_id=server['id']) msg = ('Port with mac address {} is expected to be part of ' 'offloaded flows') for port in ports['ports']: if 'capabilities' in port['binding:profile'] and 'switchdev'\ in port['binding:profile']['capabilities']: self.assertIn(port['mac_address'], out, msg.format(port['mac_address'])) # Pings are running check flows exist # Retrieve all hypvervisors hypervisors = self._get_hypervisor_ip_from_undercloud( shell='/home/stack/stackrc') # Command to check offloaded flows in OVS cmd = 'sudo ovs-appctl dpctl/dump-flows type=offloaded' for hypervisor in hypervisors: out = shell_utils.run_command_over_ssh(hypervisor, cmd) msg = 'Hypervisor {} has no offloaded flows in OVS'.format( hypervisor) self.assertNotEmpty(out, msg) LOG.info('Hypercisor {} has offloaded flows in OVS'.format( hypervisor)) # send stop statistics signal shell_utils.stop_continuous_ping()
def _get_dpdk_nics_mapping(self, hypervisor_ip, mac): cmd = "sudo cat /var/lib/os-net-config/dpdk_mapping.yaml" output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) dpdk_nics_map = yaml.safe_load(output) for dpdk_nic_map in dpdk_nics_map: if dpdk_nic_map['mac_address'] == mac: return dpdk_nic_map else: msg = ("Unable to determine DPDK NIC Mapping for " "MAC: '%(mac)s'" % { 'mac': mac }) raise Exception(msg)
def test_igmp_snooping_deployment(self, test='igmp_snooping_deployment'): """Check that igmp snooping bonding is properly configure mcast_snooping_enable and mcast-snooping-disable-flood-unregistered configured in br-int """ LOG.info('Starting {} test.'.format(test)) hypervisors = self._get_hypervisor_ip_from_undercloud( shell='/home/stack/stackrc') result = [] cmd = 'sudo ovs-vsctl --format=json list bridge br-int' checks = { 'mcast_snooping_enable': True, 'mcast-snooping-disable-flood-unregistered': 'true' } for hypervisor_ip in hypervisors: output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) # ovs command returns boolean in small letters ovs_data = json.loads(output) ovs_data_filt = {} try: ovs_data_filt['mcast_snooping_enable'] = \ (ovs_data['data'][0] [ovs_data['headings'].index('mcast_snooping_enable')]) ovs_data_filt['mcast-snooping-disable-flood-unregistered'] = \ (dict(ovs_data['data'][0] [ovs_data['headings'].index('other_config')][1]) ['mcast-snooping-disable-flood-unregistered']) except Exception: pass diff_checks_cmd = (set(checks.keys()) - set(ovs_data_filt.keys())) if len(diff_checks_cmd) > 0: result.append( "{}. Missing checks: {}. Check ovs cmd output".format( hypervisor_ip, ', '.join(diff_checks_cmd))) for check in checks: if check not in diff_checks_cmd: if ovs_data_filt[check] != checks[check]: msg = ("{}. Check failed: {}. Expected: {} - Found: {}" .format(hypervisor_ip, check, checks[check], ovs_data_filt[check])) result.append(msg) self.assertTrue(len(result) == 0, '. '.join(result)) return True
def test_igmp_restart_ovs(self, test='igmp_restart_ovs'): """Test restart ovs Check that multicast configuration is not lost after ovs restart. Restart ovs and then execute test_igmp_snooping_deployment """ LOG.info('Starting {} test.'.format(test)) network_backend = self.discover_deployment_network_backend() hypervisor_ips = self._get_hypervisor_ip_from_undercloud() ovs_cmd = 'sudo systemctl restart openvswitch.service' for hyp in hypervisor_ips: shell_utils.run_command_over_ssh(hyp, ovs_cmd) if network_backend == 'ovn': ovn_cmd = 'sudo systemctl restart tripleo_ovn_controller.service' controller_ips = shell_utils.get_controllers_ip_from_undercloud( shell=CONF.nfv_plugin_options.undercloud_rc_file) # We assume that controller nodes act as ovn controllers for node in controller_ips: shell_utils.run_command_over_ssh(hyp, ovn_cmd) self.test_igmp_snooping_deployment() # Give time to have everything up after reboot so that other testcases # executed after this one do not fail time.sleep(60)
def _get_node_nfv_status(self, hypervisor_ip): dpdk_status = False sriov_status = False cmd = ("sudo ovs-vswitchd --version | " "awk '{ if ($1 == \"DPDK\") print 1; }';" "echo '|';sudo cat /etc/puppet/hieradata/service_names.json") output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) if output: params = output.split('|') if params: if '1' in params[0].strip('\n'): dpdk_status = True service_names = json.loads(params[1]) if (service_names['service_names'] and "neutron_sriov_agent" in service_names['service_names']): sriov_status = True return dpdk_status, sriov_status
def _get_cpu_details(self, hypervisor_ip): cmd = "sudo lscpu | grep 'Model name';sudo lscpu | grep 'Flags'" output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) if output: cpu_model = "" cpu_flags = [] params = output.split('\n') if params: for param in params: if "Model name" in param: cpu_model = param.split(':')[1].strip(' \n') elif "Flags" in param: cpu_flags = param.split(':')[1].strip(' \n').split(' ') return cpu_model, cpu_flags else: msg = "Unable to determine 'CPU Model name'" raise Exception(msg)
def _get_physical_memory(self, hypervisor_ip): mem_total_kb = 0 cmd = "sudo dmidecode --type memory | grep 'Size' | grep '[0-9]'" output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) for line in output.split('\n'): if line: mem_info = line.split(':')[1].strip() mem_val = mem_info.split(' ') mem_unit = mem_val[1].strip(' ').lower() if mem_unit == 'kb': memory_kb = int(mem_val[0].strip(' ')) elif mem_unit == 'mb': memory_kb = (int(mem_val[0].strip(' ')) * 1024) elif mem_unit == 'gb': memory_kb = (int(mem_val[0].strip(' ')) * 1024 * 1024) mem_total_kb += memory_kb return (mem_total_kb / 1024)
def ping_via_network_namespace(self, ping_to_ip, network_id): cmd = ("sudo ip netns exec qdhcp-" + network_id + " ping -c 10 " + ping_to_ip) ctrl_ip = urlparse(CONF.identity.uri).netloc.split(':')[0] result = shell_utils.run_command_over_ssh(ctrl_ip, cmd) for line in result.split('\n'): if 'packets transmitted' in line: LOG.info("Ping via namespace result: %s", line) received_str = line.split(',')[1].strip() try: received = int(received_str.split(' ')[0]) except ValueError: break if received > 0: return True break return False
def get_ovs_interface_statistics(self, interfaces, previous_stats=None, hypervisor=None): """This method get ovs interface statistics :param interfaces: interfaces in which statistics will be retrieved :param previous_stats: get the difference between current stats and previous stats :param hypervisor: hypervisor ip, if None it will be selected the first one :return statistics """ self.ip_address = self._get_hypervisor_ip_from_undercloud( **{'shell': '/home/stack/stackrc'}) hypervisor_ip = self.ip_address[0] if hypervisor is not None: if hypervisor not in self.ip_address: raise ValueError('invalid hypervisor ip {}, not in {}' .format(hypervisor, ' '.join(self.ip_address))) else: hypervisor_ip = hypervisor shell_utils.check_pid_ovs(hypervisor_ip) # We ensure that a number is being parsed, otherwise we fail statistics = {} for interface in interfaces: command = 'sudo ovs-vsctl get Interface {} ' \ 'statistics'.format(interface) statistics[interface] = \ yaml.safe_load( shell_utils.run_command_over_ssh( hypervisor_ip, command).replace( '"', '').replace( '{', '{"').replace(', ', ', "').replace('=', '":')) if previous_stats is not None and \ interface in previous_stats.keys(): for stat in statistics[interface].keys(): if stat in previous_stats[interface].keys(): statistics[interface][stat] -= \ previous_stats[interface][stat] else: raise ValueError('missing ovs interface stat {} ' 'to compare'.format(stat)) return statistics
def test_offload_nic_eswitch_mode(self, test='offload'): """Check eswitch mode of nic for offload on all hypervisors :param test: Test name from the external config file. """ test_dict = self.test_setup_dict[test] if 'offload_nics' in test_dict: offload_nics = test_dict['offload_nics'] else: raise ValueError('offload_nics is not defined in offload test') # Retrieve all hypvervisors hypervisors = self._get_hypervisor_ip_from_undercloud( shell='/home/stack/stackrc') # ethtool cmd to retrieve PCI bus of interface ethtool_cmd = ("sudo ethtool -i {} | grep bus-info " "| cut -d ':' -f 2,3,4 | awk '{{$1=$1}};1'") # devlink cmd to retrieve switch mode of interface devlink_cmd = "sudo devlink dev eswitch show pci/{}" # Intialize results list result = [] # Expected result is a list of dicts containing a dict of # hypervisor's IP, its offload nics as keys and the value 'true' # Example: # [{'192.0.160.1': [{'p6p1': 'true'}, {'p6p2': 'true'}]}, # {'192.0.160.2': [{'p6p1': 'true'}, {'p6p2': 'true'}]}] expected_result = [{ip: [{nic: 'true'} for nic in offload_nics]} for ip in hypervisors] for hypervisor in hypervisors: dev_result = [] for nic in offload_nics: pci = shell_utils.run_command_over_ssh(hypervisor, ethtool_cmd.format(nic)) dev_query = shell_utils.\ run_command_over_ssh(hypervisor, devlink_cmd.format(pci)) if 'switchdev' in dev_query: output = 'true' else: output = 'false' LOG.info("Hypervisor '{h}' NIC '{n}' is in switchdev mode: {r}" .format(h=hypervisor, n=nic, r=output)) dev_result.append({nic: output}) result.append({hypervisor: dev_result}) msg = "Not all hypervisors contains nics in switchev mode" self.assertItemsEqual(expected_result, result, msg)
def get_osp_release(self, hypervisor=None): """Gather OSP release Takes the OSP release from the hypervisor :param hypervisor: Ip of the hypervisor to work on (optional) :return OSP version integer """ if not hypervisor: hyper_kwargs = {'shell': '/home/stack/stackrc'} hypervisor = self._get_hypervisor_ip_from_undercloud( **hyper_kwargs)[0] ver = shell_utils.\ run_command_over_ssh(hypervisor, 'cat /etc/rhosp-release') if ver == '': ver = shell_utils.run_command_over_ssh( hypervisor, 'cat /var/lib/rhos-release/latest-installed') return int(re.findall(r'\d+', ver)[0])
def validate_no_reboot_in_stack_update(self, stack_name='overcloud', hypervisors_ip=False): """test node didn't reboot meanwhile stack update quries heat api and validate that no reboot occuered durring stack update """ LOG.info('Started validateing no reboot meanwhile stack update.') LOG.info('Fetching stack update start and end from heat API') for event in self.os_client\ .undercloud_heatclient.events.list(stack_name): if event.resource_status_reason == 'Stack UPDATE started': update_start = datetime.datetime.strptime( event.event_time, '%Y-%m-%dT%H:%M:%SZ') elif event.resource_status_reason ==\ 'Stack UPDATE completed successfully': update_end = datetime.datetime.strptime( event.event_time, '%Y-%m-%dT%H:%M:%SZ') if not hypervisors_ip: LOG.info('Fetching overcloud hypervisors ip addresses') hypervisors_ip = self._get_hypervisor_ip_from_undercloud() rebooted_hypervisors = [] for hypervisor in hypervisors_ip: try: last_reboot = datetime.datetime.strptime( shell_utils.run_command_over_ssh(hypervisor, 'uptime -s'), '%Y-%m-%d %H:%M:%S\n') except NoValidConnectionsError: LOG.info('One or more of the hypervisor is ' 'unreachable via ssh please make sure all ' 'hypervisors are up') raise NoValidConnectionsError if last_reboot <= update_end and last_reboot >= update_start: rebooted_hypervisors.append(hypervisor) self.assertEmpty( rebooted_hypervisors, 'Computes with the following {} ip address rebooted ' 'durring the update'.format(rebooted_hypervisors))
def validate_kargs(self, ip): """validates kernal args are as expected :param ip: ip adress of a server to validate """ LOG.info('Validating kargs are competable') # TODO(eshulman) replace with a heat query expected_args = CONF.nfv_plugin_options.kernel_args.split(' ') try: cmdline = shell_utils.run_command_over_ssh(ip, 'cat /proc/cmdline')\ .split(' ') except Exception as err: import sys self.exec_info = sys.exc_info() raise err for arg in expected_args: self.assertIn(arg, cmdline, 'kernel arguments did not update after node reboot')
def get_ovs_multicast_groups(self, switch, multicast_ip=None, hypervisor=None): """This method get ovs multicast groups :param switch: ovs switch to get multicast groups :param multicast_ip: filter by multicast ip :param hypervisor: hypervisor ip, if None it will be selected the first one :return multicast groups """ self.ip_address = self._get_hypervisor_ip_from_undercloud( **{'shell': '/home/stack/stackrc'}) hypervisor_ip = self.ip_address[0] if hypervisor is not None: if hypervisor not in self.ip_address: raise ValueError('invalid hypervisor ip {}, not in {}' .format(hypervisor, ' '.join(self.ip_address))) else: hypervisor_ip = hypervisor shell_utils.check_pid_ovs(hypervisor_ip) command = 'sudo ovs-appctl mdb/show {}'.format(switch) output = list(filter(None, shell_utils.run_command_over_ssh( hypervisor_ip, command).split('\n'))) fields = None output_data = [] for line in output: data = list(filter(None, line.split(" "))) if fields is None: fields = data else: data = dict(zip(fields, data)) if multicast_ip is None or \ multicast_ip is not None and data['GROUP'] == multicast_ip: output_data.append(data) return output_data
def check_number_queues(self): """This method checks the number of max queues""" self.ip_address = self._get_hypervisor_ip_from_undercloud( **{'shell': '/home/stack/stackrc'}) ovs_process_pid = shell_utils.check_pid_ovs(self.ip_address[0]) count_pmd = "ps -T -p {} | grep pmd | wc -l".format(ovs_process_pid) numpmds = int(shell_utils.run_command_over_ssh(self.ip_address[0], count_pmd)) # We ensure that a number is being parsed, otherwise we fail cmd = r'sudo ovs-vsctl show | sed -n "s/.*n_rxq=.\([1-9]\).*/\\1/p"' numqueues = (shell_utils. run_command_over_ssh(self.ip_address[0], cmd)).encode('ascii', 'ignore') if not isinstance(numqueues, type(str())): numqueues = numqueues.decode("utf-8") msg = "There are no queues available" self.assertNotEqual((numqueues.rstrip("\n")), '', msg) # Different multiple queues is not a supported scenario as per now self.assertTrue(str.isdigit(numqueues.split("\n")[0]), "Queue recieved is not a digit") numqueues = int(numqueues.split("\n")[0]) maxqueues = numqueues * numpmds return maxqueues
def _get_nodes_cores_info(self, hypervisor_ip): dict_cpus = {} cmd = "sudo lscpu -p=NODE,CORE,CPU | grep -v ^#" output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) for line in output.split('\n'): if line: cpu_info = line.split(',') node = int(cpu_info[0]) cpu = int(cpu_info[1]) thread = int(cpu_info[2]) # CPU and NUMA node together forms a unique value, as cpu is # specific to a NUMA node # NUMA node id and cpu id tuple is used for unique key dict_key = node, cpu if dict_key in dict_cpus: if thread not in dict_cpus[dict_key]['thread_siblings']: dict_cpus[dict_key]['thread_siblings'].append(thread) else: cpu_item = {} cpu_item['thread_siblings'] = [thread] cpu_item['cpu'] = cpu cpu_item['numa_node'] = node dict_cpus[dict_key] = cpu_item return dict_cpus
def test_igmp_snooping_deployment(self, test='igmp_snooping_deployment'): """Check that igmp snooping bonding is properly configure mcast_snooping_enable and mcast-snooping-disable-flood-unregistered configured in br-int """ LOG.info('Starting {} test.'.format(test)) network_backend = self.discover_deployment_network_backend() if network_backend == 'ovs': hypervisors = self._get_hypervisor_ip_from_undercloud() result = [] cmd = 'sudo ovs-vsctl --format=json list bridge br-int' checks = {'mcast_snooping_enable': True, 'mcast-snooping-disable-flood-unregistered': True} for hypervisor_ip in hypervisors: output = shell_utils.run_command_over_ssh(hypervisor_ip, cmd) # ovs command returns boolean in small letters ovs_data = json.loads(output) ovs_data_filt = {} try: ovs_data_filt['mcast_snooping_enable'] = \ (ovs_data['data'][0] [ovs_data['headings'].index('mcast_snooping_enable')]) ovs_data_filt['mcast-snooping-disable' '-flood-unregistered'] = \ (dict(ovs_data['data'][0] [ovs_data['headings'].index('other_config')][1]) ['mcast-snooping-disable-flood-unregistered']) except Exception: pass diff_checks_cmd = (set(checks.keys()) - set(ovs_data_filt.keys())) if len(diff_checks_cmd) > 0: result.append("{}. Missing checks: {}. Check ovs cmd " "output".format(hypervisor_ip, ', '.join(diff_checks_cmd))) for check in checks: if check in diff_checks_cmd: if type(ovs_data_filt[check]) == str: # If object is not equal to 'true' or 'false' # ValueError exception will be raised ovs_data_filt[check] = \ strtobool(ovs_data_filt[check]) if ovs_data_filt[check] != checks[check]: msg = ("{}. Check failed: {}. Expected: {} " "- Found: {}" .format(hypervisor_ip, check, checks[check], ovs_data_filt[check])) result.append(msg) self.assertTrue(len(result) == 0, '. '.join(result)) # We assume that controller nodes act as ovn controllers elif network_backend == 'ovn': controller = shell_utils.get_controllers_ip_from_undercloud( shell=CONF.nfv_plugin_options.undercloud_rc_file)[0] # Configuration should be identical across controllers igmp_configured = shell_utils.get_value_from_ini_config( controller, '/var/lib/config-data/puppet-generated' '/neutron/etc/neutron/neutron.conf', 'ovs', 'igmp_snooping_enable') self.assertTrue(igmp_configured, 'IGMP not enabled in deployment') ovn_logical_switches_cmd = ('sudo podman exec -it ovn_controller' ' ovn-nbctl list Logical_Switch') ovn_logical_switches_output = shell_utils.run_command_over_ssh( controller, ovn_logical_switches_cmd) # We expect to have at least a single logical switch if '_uuid 'not in ovn_logical_switches_output: raise ValueError('Failed to query OVN northbound DB' ', no logical switch info was returned') re_igmp_string = \ r'mcast_snoop="true"' pattern = re.compile(re_igmp_string) igmp_lines = pattern.findall(ovn_logical_switches_output) LOG.info('Located {} logical switches with IGMP enabled' .format(len(igmp_lines))) msg = "IGMP is not enabled on any logical switch" self.assertNotEmpty(igmp_lines, msg) re_flood_string = \ r'mcast_flood_unregistered="\w+"' flood_pattern = re.compile(re_flood_string) flood_lines = \ flood_pattern.findall(ovn_logical_switches_cmd) for line in flood_lines: if 'true' in line: LOG.warning("Located a logical switch with " "'mcast_flood_unregistered' set to 'true', " "this is not optimal and will potentially" "cause multicast to behave as broadcast. " "This setting should be set to 'false'") else: raise ValueError("Network backend '{}' is not supported" .format(network_backend)) return True
def test_check_igmp_reports(self, test='check_igmp_reports'): """Check igmp reports are forwarded Check igmp reports are forwarded from the vms and arrive to the switch. It will be check the internal bridge in which it is the nic connected to the external switch Tests https://bugzilla.redhat.com/show_bug.cgi?id=1933734 """ LOG.info('Starting {} test.'.format(test)) if self.external_resources_data is None: raise ValueError('External resource data is required for the test') servers, key_pair = self.create_and_verify_resources(test=test) igmp_reports = CONF.nfv_plugin_options.igmp_reports reports_interface = igmp_reports['reports_interface'] tcpdump_file = tempfile.NamedTemporaryFile().name test_server = servers[0] # Check tcpdump is installed cmd_check_dump = "PATH=$PATH:/usr/sbin; which tcpdump " \ "2>/dev/null || true" LOG.info('Executed on {}: {}'.format( test_server['hypervisor_ip'], cmd_check_dump)) output = shell_utils.run_command_over_ssh(test_server['hypervisor_ip'], cmd_check_dump) self.assertNotEqual(output, '', "tcpdump not installed in {}".format( test_server['hypervisor_ip'])) # It will generate the join/leave igmp messages cmd_mcast = "sudo timeout 3 python " \ "/usr/local/bin/multicast_traffic.py -r -g 239.1.1.1 " \ "-p 5000 -c 1 || true" # Capture all igmp messages cmd_dump = "PATH=$PATH:/usr/sbin; sudo tcpdump -i {} igmp " \ "> {} 2>&1 &".format(reports_interface, tcpdump_file) # Filter only igmp reports messages (join/leave), not igmp queries cmd_result = "sudo killall -SIGINT tcpdump;cat {} | " \ "( grep igmp || true ) | ( grep report || true ) | " \ "wc -l".format(tcpdump_file) ssh_source = self.get_remote_client(test_server['fip'], username=self. instance_user, private_key=key_pair[ 'private_key']) LOG.info('Executed on {}: {}'.format( test_server['hypervisor_ip'], cmd_dump)) shell_utils.run_command_over_ssh(test_server['hypervisor_ip'], cmd_dump) LOG.info('Executed on {}: {}'.format(test_server['fip'], cmd_mcast)) ssh_source.exec_command(cmd_mcast) LOG.info('Executed on {}: {}'.format(test_server['hypervisor_ip'], cmd_mcast)) output = int(shell_utils.run_command_over_ssh( test_server['hypervisor_ip'], cmd_result)) self.assertGreater(output, 0, "No igmp reports received") LOG.info('Igmp reports being forwarded properly')
def test_hypervisor_tuning(self, test='hypervisor_tuning'): """Test tuning state of hypervisor Test the following states: - Packages (given in config) - Active services (given in config) - Tuned active profile (given in config) - Kernel arguments (given in config) """ tuning_details = \ json.loads(CONF.nfv_plugin_options.hypervisor_tuning_details) packages = tuning_details.get("packages") services = tuning_details.get("services") tuned_profiles = tuning_details.get("tuned_profiles") kernel_args = tuning_details.get("kernel_args") self.hypervisor_ip = self._get_hypervisor_ip_from_undercloud()[0] self.assertNotEmpty(self.hypervisor_ip, "No hypervisor found") test_result = [] if packages: pkg_check = "rpm -qa | grep" for package in packages: tmpl = " -e ^{}" pkg_check += tmpl.format(package) result = shell_utils.run_command_over_ssh(self.hypervisor_ip, pkg_check).split() if result: for pkg in packages: if not fnmatch.filter(result, pkg): test_result.append( "Missing required packages. " "Found following packages: {}".format(result)) LOG.info("Found the following packages: {}".format(result)) else: test_result.append("Packages: no output received") if services: svc_check = "systemctl is-active" for service in services: svc_check += " {}".format(service) result = shell_utils.run_command_over_ssh(self.hypervisor_ip, svc_check).strip('\n') if result: if result.split('\n').count('active') != len(services): test_result.append("Some of the requested services are " "not in an active state.") LOG.info('The services states - {}'.format( list(zip(services, result.split('\n'))))) else: test_result.append("Services: no output received") if tuned_profiles: cmd = "sudo tuned-adm active | awk '{print $4}'" result = shell_utils.run_command_over_ssh(self.hypervisor_ip, cmd).strip('\n') if result not in tuned_profiles: test_result.append( "Tuned {0} profile is not Active".format(tuned_profiles)) if kernel_args: grub_output_cmd = "sudo cat /proc/cmdline" result = shell_utils.run_command_over_ssh(self.hypervisor_ip, grub_output_cmd) if result: for arg in kernel_args: if arg not in result: test_result.append( "The kernel args are missing - {}".format(arg)) else: test_result.append("Kernel args: no output received") test_result = '\n'.join(test_result) self.assertEmpty(test_result, test_result)
def learn_queues(self, servers, key_pair): """learn about queues Learn about queues: * mapping physical/virtual queues * rate/cpu params :param servers: server dict with vms :param key_pair: key pair """ learning_config = CONF.nfv_plugin_options.multiqueue_learning if not learning_config["learn"]: LOG.info('Skipping multiqueue learning due to configuration.') return LOG.info('Starting multiqueue learning') # set affinity to be sure that there are no 2 physical queues in the # same pmd which causes that one virtual queue will be used instead of # 2 and it will not be able to map one physical queue with one # virtual queue for interface in learning_config["pmd_rxq_affinity"]: cmd_affinity = "sudo ovs-vsctl set Interface {} " \ "other_config:pmd-rxq-affinity=\"{}\"".\ format(interface["interface"], interface["pmd_rxq_affinity"]) LOG.info('learn_queues cmd {}'.format(cmd_affinity)) shell_utils.run_command_over_ssh( servers['testpmd']['hypervisor_ip'], cmd_affinity) # training cmd cmd_training = "{} --action gen_traffic --pps \"{}\" --traffic_json" \ " {} --duration {} --multiplier {}".\ format(learning_config["injector"], learning_config["pps"], learning_config["queues_json"], learning_config["duration"], learning_config["multiplier"]) LOG.info('learn_queues cmd {}'.format(cmd_training)) servers['trex']['ssh_source'].exec_command(cmd_training) # get pmd stats cmd_pmd_rxq_show = "sudo ovs-appctl dpif-netdev/pmd-rxq-show" LOG.info('learn_queues cmd {}'.format(cmd_pmd_rxq_show)) pmd_rxq_output = shell_utils.run_command_over_ssh( servers['testpmd']['hypervisor_ip'], cmd_pmd_rxq_show) LOG.info('learn_queues queues: {}'.format(pmd_rxq_output)) # remove configure affinity before after the learning for interface in learning_config["pmd_rxq_affinity"]: cmd_affinity = "sudo ovs-vsctl remove Interface {} " \ "other_config pmd-rxq-affinity".\ format(interface["interface"]) LOG.info('learn_queues cmd {}'.format(cmd_affinity)) shell_utils.run_command_over_ssh( servers['testpmd']['hypervisor_ip'], cmd_affinity) # copy pmd file to trex vm with tempfile.NamedTemporaryFile() as fp: fp.write(pmd_rxq_output.encode()) fp.flush() self.copy_file_to_remote_host(servers['trex']['fip'], key_pair['private_key'], self.instance_user, files=os.path.basename(fp.name), src_path=os.path.dirname(fp.name), dst_path=os.path.dirname(fp.name), timeout=60) # parse pmd file and update queues.json file cmd_pmd_parse = "{} --action parse_pmd_stats --pmd_stats {} " \ "--traffic_json {} --pps \"{}\"".\ format(learning_config["injector"], fp.name, learning_config["queues_json"], learning_config["pps"]) LOG.info('learn_queues cmd {}'.format(cmd_pmd_parse)) servers['trex']['ssh_source'].exec_command(cmd_pmd_parse) LOG.info('Multiqueue learning finished')