def restart_ovs_on_computes(self, env, os_conn): ovs_agent_ids, ovs_conroller_agents = self.get_ovs_agents(env, os_conn) os_conn.wait_agents_alive(ovs_agent_ids) common.disable_ovs_agents_on_controller(env) os_conn.wait_agents_down(ovs_conroller_agents) common.restart_ovs_agents_on_computes(env) common.enable_ovs_agents_on_controllers(env) os_conn.wait_agents_alive(ovs_agent_ids) # sleep to make sure that system will be stable after ovs restarting time.sleep(30)
def test_ovs_restart_pcs_disable_enable_ping_private_vms(self): """Restart openvswitch-agents with pcs disable/enable on controllers. Steps: 1. Update default security group if needed 2. Create CONFIG 1: Network: test_net_05 SubNetw: test_net_05__subnet, 192.168.5.0/24 Router: test_router_05 3. Create CONFIG 2: Network: test_net_06 SubNetw: test_net_06__subnet, 192.168.6.0/24 Router: test_router_06 4. Launch 'test_vm_05' inside 'config 1' 5. Launch 'test_vm_06' inside 'config 2' 6. Go to 'test_vm_05' console and send pings to 'test_vm_05'. Pings should NOT go between VMs. 7. Operations with OVS agents: - Check that all OVS are alive; - Disable ovs-agents on all controllers; - Check that they wend down; - Restart OVS agent service on all computes; - Enable ovs-agents on all controllers; - Check that they wend up and alive; 8. Wait 30 seconds, send pings from 'test_vm_05' to 'test_vm_06' and check that they are still NOT successful. Duration 5m """ self._prepare_openstack() # Check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) # Disable ovs agent on all controllers common.disable_ovs_agents_on_controller(self.env) # Then check that all ovs went down self.os_conn.wait_agents_down(self.ovs_conroller_agents) # Restart ovs agent service on all computes common.restart_ovs_agents_on_computes(self.env) # Enable ovs agent on all controllers common.enable_ovs_agents_on_controllers(self.env) # Then check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) # sleep is used to check that system will be stable for some time # after restarting service time.sleep(30) self.check_no_ping_from_vm(self.server1, self.instance_keypair, self.server2_ip, timeout=None)
def test_ovs_restart_pcs_disable_enable(self, count): """Restart openvswitch-agents with pcs disable/enable on controllers Steps: 1. Update default security group 2. Create router01, create networks net01: net01__subnet, 192.168.1.0/24, net02: net02__subnet, 192.168.2.0/24 and attach them to router01. 3. Launch vm1 in net01 network and vm2 in net02 network on different computes 4. Go to vm1 console and send pings to vm2 5. Disable ovs-agents on a controller, restart service neutron-plugin-openvswitch-agent on all computes, and enable them back. To do this, launch the script against master node. 6. Wait 30 seconds, send pings from vm1 to vm2 and check that it is successful. 7. Repeat steps 6-7 'count' argument times Duration 10m """ self._prepare_openstack() for _ in range(count): # Check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) # Disable ovs agent on a controller common.disable_ovs_agents_on_controller(self.env) # Then check that all ovs went down self.os_conn.wait_agents_down(self.ovs_conroller_agents) # Restart ovs agent service on all computes common.restart_ovs_agents_on_computes(self.env) # Enable ovs agent on a controller common.enable_ovs_agents_on_controllers(self.env) # Then check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) # sleep is used to check that system will be stable for some time # after restarting service time.sleep(30) network_checks.check_ping_from_vm( self.env, self.os_conn, self.server1, self.instance_keypair, self.server2_ip, timeout=10 * 60) # check all agents are alive assert all([agt['alive'] for agt in self.os_conn.neutron.list_agents()['agents']])
def test_ovs_new_flows_added_after_restart(self): """Check that new flows are added after ovs-agents restart Steps: 1. Create network net01: net01__subnet, 192.168.1.0/24 2. Launch vm1 in net01 network 3. Get list of flows for br-int 4. Save cookie parameter for bridge 5. Disable ovs-agents on all controllers, restart service neutron-plugin-openvswitch-agent on all computes, and enable them back. To do this, launch the script against master node. 6. Check that all ovs-agents are in alive state 7. Get list of flows for br-int again 8. Compare cookie parameters """ self._prepare_openstack() server = self.os_conn.nova.servers.find(name="server_for_flow_check") node_name = getattr(server, "OS-EXT-SRV-ATTR:hypervisor_hostname") compute = [i for i in self.env.get_nodes_by_role('compute') if i.data['fqdn'] == node_name][0] # Check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) before_value = self.get_current_cookie(compute) assert all([len(x) < 2 for x in before_value.values()]) # Disable ovs agent on all controllers common.disable_ovs_agents_on_controller(self.env) # Then check that all ovs went down self.os_conn.wait_agents_down(self.ovs_conroller_agents) # Restart ovs agent service on all computes common.restart_ovs_agents_on_computes(self.env) # Enable ovs agent on all controllers common.enable_ovs_agents_on_controllers(self.env) # Then check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) # sleep is used to check that system will be stable for some time # after restarting service time.sleep(30) after_value = self.get_current_cookie(compute) assert before_value != after_value assert all([len(x) < 2 for x in after_value.values()])
def test_ovs_restart_pcs_vms_on_single_compute_in_single_network(self): """Check connectivity for instances scheduled on a single compute in a single private network Steps: 1. Update default security group 2. Create networks net01: net01__subnet, 192.168.1.0/24 3. Launch vm1 and vm2 in net01 network on a single compute compute 4. Go to vm1 console and send pings to vm2 5. Disable ovs-agents on all controllers, restart service neutron-plugin-openvswitch-agent on all computes, and enable them back. To do this, launch the script against master node. 6. Wait 30 seconds, send pings from vm1 to vm2 and check that it is successful. Duration 10m """ self._prepare_openstack() # Check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) # Disable ovs agent on all controllers common.disable_ovs_agents_on_controller(self.env) # Then check that all ovs went down self.os_conn.wait_agents_down(self.ovs_conroller_agents) # Restart ovs agent service on all computes common.restart_ovs_agents_on_computes(self.env) # Enable ovs agent on all controllers common.enable_ovs_agents_on_controllers(self.env) # Then check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) # sleep is used to check that system will be stable for some time # after restarting service time.sleep(30) network_checks.check_ping_from_vm( self.env, self.os_conn, self.server1, self.instance_keypair, self.server2_ip, timeout=3 * 60) # check all agents are alive assert all([agt['alive'] for agt in self.os_conn.neutron.list_agents()['agents']])
def test_ovs_restart_with_iperf_traffic(self): """Checks that iperf traffic is not interrupted during ovs restart Steps: 1. Run iperf server on server2 2. Run iperf client on server 1 3. Check that packet losses < 1% 4. Disable ovs-agents on all controllers, restart service neutron-plugin-openvswitch-agent on all computes, and enable them back. 5. Check that all ovs-agents are in alive state 6. Check that iperf traffic wasn't interrupted during ovs restart, and not more than 20% datagrams are lost """ self._prepare_openstack() iperf_log_file = '/tmp/iperf_client.log' client = self.server1 server = self.server2 # Launch iperf client pid = self.launch_iperf_client(client, server, self.instance_keypair, vm_login='******', stdout=iperf_log_file) time.sleep(60) self.wait_command_done(pid, vm=client, keypair=self.instance_keypair, vm_login='******') with self.os_conn.ssh_to_instance(self.env, vm=client, vm_keypair=self.instance_keypair, username='******') as remote: with remote.open(iperf_log_file) as f: iperf_result = f.read() # Check iperf traffic before restart lost = self.get_lost_percentage(iperf_result) err_msg = "Packet losses more than 1%. Actual value is {0}%".format( lost) assert lost < 1, err_msg self.os_conn.wait_agents_alive(self.ovs_agent_ids) # Launch client in background and restart agents pid = self.launch_iperf_client(client, server, self.instance_keypair, vm_login='******', stdout=iperf_log_file) common.disable_ovs_agents_on_controller(self.env) self.os_conn.wait_agents_down(self.ovs_conroller_agents) common.restart_ovs_agents_on_computes(self.env) common.enable_ovs_agents_on_controllers(self.env) self.os_conn.wait_agents_alive(self.ovs_agent_ids) self.wait_command_done(pid, vm=client, keypair=self.instance_keypair, vm_login='******') with self.os_conn.ssh_to_instance(self.env, vm=client, vm_keypair=self.instance_keypair, username='******') as remote: with remote.open(iperf_log_file) as f: iperf_result = f.read() err_msg = "{0}% datagrams lost. Should be < 20%".format(lost) assert lost < 20, err_msg # check all agents are alive assert all([agt['alive'] for agt in self.os_conn.neutron.list_agents()['agents']])
def test_restart_openvswitch_agent_under_bat(self): """Restart openvswitch-agents with broadcast traffic background Steps: 1. Go to vm1's console and run arping to initiate broadcast traffic: arping -I eth0 <vm2_fixed_ip> 2. Disable ovs-agents on all controllers 3. Restart service 'neutron-plugin-openvswitch-agent' on all computes 4. Enable ovs-agents back. 5. Check that pings between vm1 and vm2 aren't interrupted or not more than 2 packets are lost """ self._prepare_openstack() # Run arping in background on server01 towards server02 srv_list = self.os_conn.nova.servers.list() srv1, srv2 = srv_list[:2] vm_ip = self.os_conn.get_nova_instance_ips(srv2)['fixed'] arping_cmd = 'sudo /usr/sbin/arping -I eth0 {}'.format(vm_ip) arping_log = '/tmp/arp.log' with self.os_conn.ssh_to_instance(self.env, srv1, vm_keypair=self.instance_keypair, username='******') as remote: pid = remote.background_call(arping_cmd, stdout=arping_log) # Then check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) # Disable ovs agent on all controllers common.disable_ovs_agents_on_controller(self.env) # Then check that all ovs went down self.os_conn.wait_agents_down(self.ovs_conroller_agents) # Restart ovs agent service on all computes common.restart_ovs_agents_on_computes(self.env) # Enable ovs agent on all controllers common.enable_ovs_agents_on_controllers(self.env) # Then check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) # Check that arping is still executing cmd = 'ps -o pid | grep {}'.format(pid) with self.os_conn.ssh_to_instance(self.env, srv1, vm_keypair=self.instance_keypair, username='******') as remote: result = remote.execute(cmd) assert result.is_ok, 'Arping command is died' # Read log of arpping execution for future possible debug with remote.open(arping_log) as f: logger.debug(f.read()) # Check connectivity network_checks.check_vm_connectivity(self.env, self.os_conn)