def test_vms_with_cpu_thread_policy_wo_hyper_threading( self, env, os_conn, hosts_without_hyper_threading, flavors, networks, keypair, security_group): """This test checks vms with cpu_thread_policy parameter in case of disabled hyper-threading Steps: 1. Create net1 with subnet and router1 with interface to net1 2. Create cpu pinning flavors with hw:numa_nodes=1 and cpu_thread_policy 3. Boot vm and check that all vcpus are on the different core 4. Redo for all flavors 5. Check vms connectivity """ host = hosts_without_hyper_threading[0] zone = 'nova:{}'.format(host) for flv in flavors: vm = os_conn.create_server(name='vm{}'.format(flv.name), flavor=flv.id, key_name=keypair.name, nics=[{'net-id': networks[0]}], security_groups=[security_group.id], availability_zone=zone) used_ts_list = self.get_vm_thread_siblings_lists(os_conn, vm) assert len(used_ts_list) == flv.vcpus, ( "vcpus should be on the different cores") network_checks.check_vm_connectivity(env, os_conn)
def test_shutdown_primary_controller_with_l3_agt(self): """[Neutron VLAN and VXLAN] Shut down primary controller and check l3-agent Steps: 1. Check on what agents is router1: neutron l3-agent-list-hosting-router router1 2. If there isn't agent on the primary controller: neutron l3-agent-router-remove non_on_primary_agent_id router1 neutron l3-agent-router-add on_primary_agent_id router1 3. Destroy primary controller virsh destroy <primary_controller> 4. Wait some time until all agents are up neutron-agent-list 5. Check that all routers reschedule from primary controller: neutron router-list-on-l3-agent <on_primary_agent_id> 6. Boot vm3 in network1 7. ping 8.8.8.8 from vm3 8. ping between vm1 and vm3 by internal ip 9. ping between vm1 and vm2 by floating ip """ self._prepare_openstack() # Get current L3 agent on router01 l3_agent = self.os_conn.neutron.list_l3_agent_hosting_routers( self.router['id'])['agents'][0] # Check if the agent is not on the primary controller # Reschedule if needed if l3_agent['host'] != self.primary_host: self.os_conn.reschedule_router_to_primary_host(self.router['id'], self.primary_host) l3_agent = self.os_conn.neutron.list_l3_agent_hosting_routers( self.router['id'])['agents'][0] # virsh destroy of the primary controller self.env.destroy_nodes([self.primary_node]) # Excluding the id of the l3_agent from the list # since it will stay on the destroyed controller # and remain disabled self.l3_agent_ids.remove(l3_agent['id']) # Then check that the rest l3 agents are alive self.os_conn.wait_agents_alive(self.l3_agent_ids) # Check that there are no routers on the first agent self.check_no_routers_on_l3_agent(l3_agent['id']) # Waiting for messaging layer recovery # https://bugs.launchpad.net/mos/+bug/1592312 logger.debug('Waiting 5 minutes for messaging layer recovery') time.sleep(5 * 60) self.os_conn.add_server(self.networks[0], self.instance_keypair.name, self.hosts[0], self.security_group.id) # Create one more server and check connectivity network_checks.check_vm_connectivity(self.env, self.os_conn)
def test_l3_agent_after_drop_rabbit_port(self): """Drop rabbit port and check l3-agent work Scenario: 1. Revert snapshot with neutron cluster 2. Create network1, network2 3. Create router1 and connect it with network1, network2 and external net 4. Boot vm1 in network1 and associate floating ip 5. Boot vm2 in network2 6. Add rules for ping 7. ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other 8. with iptables in CLI drop rabbit's port #5673 on what router1 is 9. Wait for route rescheduling 10. Check that router moved to the health l3-agent 11. Boot one more VM (VM3) in network1 12. Boot vm3 in network1 13. ping 8.8.8.8, vm1 (both ip), vm2 (fixed ip) and vm3 (fixed ip) from each other Duration 10m """ # drop rabbit port self.drop_rabbit_port(router_name="router01") # check pings network_checks.check_vm_connectivity(self.env, self.os_conn)
def _prepare_openstack(self): """Prepare OpenStack for scenarios run Steps: 1. Create network1 2. Create router1 and connect it with network1 and external net 3. Boot vm1 in network1 and associate floating ip 4. Boot vm2 in network2 5. Add rules for ping 6. ping 8.8.8.8 from vm2 7. ping vm1 from vm2 and vm1 from vm2 """ # init variables exist_networks = self.os_conn.list_networks()['networks'] ext_network = [x for x in exist_networks if x.get('router:external')][0] self.zone = self.os_conn.nova.availability_zones.find(zoneName="nova") self.hosts = self.zone.hosts.keys() self.instance_keypair = self.os_conn.create_key(key_name='instancekey') self.setup_rules_for_default_sec_group() # create router self.router = self.os_conn.create_router(name="router01")['router'] self.os_conn.router_gateway_add(router_id=self.router['id'], network_id=ext_network['id']) logger.info('router {} was created'.format(self.router['id'])) # create one network by amount of the compute hosts self.net_id = self.os_conn.add_net(self.router['id']) # create two instances in that network # each instance is on the own compute for i, hostname in enumerate(self.hosts, 1): self.os_conn.create_server(name='server%02d' % i, availability_zone='{}:{}'.format( self.zone.zoneName, hostname), key_name=self.instance_keypair.name, nics=[{ 'net-id': self.net_id }]) # check pings network_checks.check_vm_connectivity(self.env, self.os_conn) # make a list of all ovs agent ids self.ovs_agent_ids = [ agt['id'] for agt in self.os_conn.neutron.list_agents( binary='neutron-openvswitch-agent')['agents'] ] # make a list of ovs agents that resides only on controllers controllers = [ node.data['fqdn'] for node in self.env.get_nodes_by_role('controller') ] ovs_agts = self.os_conn.neutron.list_agents( binary='neutron-openvswitch-agent')['agents'] self.ovs_conroller_agents = [ agt['id'] for agt in ovs_agts if agt['host'] in controllers ]
def test_vms_connectivity_after_ovs_restart_on_computes( self, env, os_conn, computes_with_dpdk_hp, flavors, networks, keypair, security_group): """This test checks connectivity between VMs with DPDK after ovs restart on computes. Please note we're not able to count DPDK huge pages only, they're added to count of 2Mb huge pages. Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Create flavor for huge pages with 512Mb ram, 1 vcpu and 1Gb disk 3. Launch vm1, vm2, vm3 on compute-1 and vm4 on compute-2, vm1 and vm2 in net1, vm3 and vm4 in net2 4. Check that neutron port has binding:vif_type = vhostuser 5. Check instance page size 6. Restart ovs on computes 7. Check vms connectivity after ovs restart """ hosts = computes_with_dpdk_hp vms_param = [(hosts[0], networks[0], None), (hosts[0], networks[0], None), (hosts[0], networks[1], None), (hosts[1], networks[1], None)] self.create_vms(os_conn, hosts, networks, flavors[0], keypair, security_group, vms_param) network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair) self.restart_ovs_on_computes(env, os_conn) network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair)
def test_cpu_pinning_one_numa_cell( self, env, os_conn, networks, flavors, security_group, aggregate): """This test checks that cpu pinning executed successfully for instances created on computes with 1 NUMA Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Launch instances vm1, vm3 in net1 with m1.small.performance on compute-1, vm2 on compute-2. 3. Check numa nodes for all vms 4. Check parameter in /etc/defaults/grub 5. Check vms connectivity """ hosts = aggregate.hosts vms = [] network_for_instances = [networks[0], networks[1], networks[0]] hosts_for_instances = [hosts[0], hosts[1], hosts[0]] cpus = get_cpu_distribition_per_numa_node(env) for i in range(2): vms.append(os_conn.create_server( name='vm{}'.format(i), flavor=flavors[0].id, nics=[{'net-id': network_for_instances[i]}], availability_zone='nova:{}'.format(hosts_for_instances[i]), security_groups=[security_group.id])) for vm in vms: host = getattr(vm, "OS-EXT-SRV-ATTR:host") assert host in hosts self.check_cpu_for_vm(os_conn, vm, 1, cpus[host]) network_checks.check_vm_connectivity(env, os_conn)
def test_cpu_pinning_resize( self, env, os_conn, networks, flavors, security_group, aggregate, aggregate_n): """This test checks that cpu pinning executed successfully for instances created on computes with 1 NUMA Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Launch vm1 using m1.small.performance-1 flavor on compute-1 and vm2 on compute-2 with m1.small.old flavor. 3. Resize vm1 to m1.small.performance-2 4. Ping vm1 from vm2 5. Resize vm1 to m1.small.performance-3 6. Ping vm1 from vm2 7. Resize vm1 to m1.small.performance-1 8. Ping vm1 from vm2 9. Resize vm1 to m1.small.old 10. Ping vm1 from vm2 11. Resize vm1 to m1.small.performance-4 12. Ping vm1 from vm2 13. Resize vm1 to m1.small.performance-1 14. Ping vm1 from vm2 """ hosts = aggregate.hosts vms = [] cpus = get_cpu_distribition_per_numa_node(env) flavors_for_resize = ['m1.small.perfomance-2', 'm1.small.perfomance-3', 'm1.small.perfomance-1', 'm1.small.old', 'm1.small.perfomance-4', 'm1.small.perfomance-1'] for i in range(2): vms.append(os_conn.create_server( name='vm{}'.format(i), flavor=flavors[i].id, nics=[{'net-id': networks[i]}], availability_zone='nova:{}'.format(hosts[i]), security_groups=[security_group.id])) vm = vms[0] for flavor in flavors_for_resize: numas = 2 for object_flavor in flavors: if object_flavor.name == flavor: vm = self.resize(os_conn, vm, object_flavor.id) break if flavor is not 'm1.small.old': if flavor in ['m1.small.perfomance-4', 'm1.small.perfomance-1']: numas = 1 host = getattr(vm, "OS-EXT-SRV-ATTR:host") assert host in hosts self.check_cpu_for_vm(os_conn, os_conn.get_instance_detail(vm), numas, cpus[host]) os_conn.wait_servers_ssh_ready(vms) network_checks.check_vm_connectivity(env, os_conn)
def test_VLAN_Allo_2M_HP_vms_req_HP_vms_with_old_flavor( self, env, os_conn, networks, nfv_flavor, keypair, security_group, volume, aggregate): """ This test checks that Huge pages set for vm1, vm2 and vm3 shouldn't use Huge pages, connectivity works properly Steps: 1. Create net01, subnet. 2. Create net 02, subnet. 3. Create router, set gateway and add interfaces to both networks 4. Launch vms using m1.small.hpgs flavor: vm1, vm3 on compute-1, vm2 on compute-2. For vm1 use new flavor, for vm2 and vm3 - old 5. Locate the part of all instances configuration that is relevant to huge pages: #on controller hypervisor=nova show hpgs-test | grep OS-EXT-SRV-ATTR:host | cut -d\| -f3 instance=nova show hpgs-test | grep OS-EXT-SRV-ATTR:instance_name | cut -d\| -f3 # on compute virsh dumpxml $instance |awk '/memoryBacking/ {p=1}; p; /\/numatune/ {p=0}' 6. ping vm2 from vm1 7. ping vm2 from vm3 8. ping vm3 from vm1 9. Check that it was allocated only HP for vm1 """ hosts = aggregate.hosts vm_0 = os_conn.create_server(name='vm1', flavor=nfv_flavor[0].id, key_name=keypair.name, nics=[{ 'net-id': networks[0] }], availability_zone='nova:{}'.format( hosts[0]), security_groups=[security_group.id]) vm_1 = os_conn.create_server(name='vm2', flavor=nfv_flavor[0].id, key_name=keypair.name, availability_zone='nova:{}'.format( hosts[1]), security_groups=[security_group.id], nics=[{ 'net-id': networks[1] }]) vm_2 = os_conn.create_server(name='vm3', flavor=nfv_flavor[0].id, key_name=keypair.name, nics=[{ 'net-id': networks[1] }], availability_zone='nova:{}'.format( hosts[1]), security_groups=[security_group.id]) vms = [vm_0, vm_1, vm_2] self.check_pages(os_conn, hosts[0], total_pages=1024, free_pages=768) self.check_pages(os_conn, hosts[1], total_pages=1024, free_pages=512) for vm in vms: self.check_instance_page_size(os_conn, vm, size=2048) network_checks.check_vm_connectivity(env, os_conn)
def test_ban_l3_agents_and_clear_last(self): """Ban all l3-agents, clear last of them and check health of l3-agent Scenario: 1. Revert snapshot with neutron cluster 2. Create network1, network2 3. Create router1 and connect it with network1, network2 and external net 4. Boot vm1 in network1 and associate floating ip 5. Boot vm2 in network2 6. Add rules for ping 7. ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other 8. Ban l3-agent on what router1 is 9. Wait for route rescheduling 10. Repeat steps 7-8 twice 11. Clear last L3 agent 12. Check that router moved to the health l3-agent 13. Boot one more VM (VM3) in network1 14. Boot vm3 in network1 15. ping 8.8.8.8, vm1 (both ip), vm2 (fixed ip) and vm3 (fixed ip) from each other Duration 10m """ net_id = self.os_conn.neutron.list_networks( name="net01")['networks'][0]['id'] devops_node = self.get_node_with_dhcp(net_id) ip = devops_node.data['ip'] # ban l3 agents for _ in range(2): self.ban_l3_agent(router_name="router01", _ip=ip) last_banned_node = self.ban_l3_agent(router_name="router01", _ip=ip, wait_for_migrate=False) # clear last banned l3 agent self.clear_l3_agent(_ip=ip, router_name="router01", node=last_banned_node, wait_for_alive=True) # create another server on net01 net01 = self.os_conn.nova.networks.find(label="net01") self.os_conn.create_server( name='server03', availability_zone='{}:{}'.format(self.zone.zoneName, self.hosts[0]), key_name=self.instance_keypair.name, nics=[{'net-id': net01.id}], security_groups=[self.security_group.id]) # check pings network_checks.check_vm_connectivity(self.env, self.os_conn)
def test_evacuate(devops_env, env, os_conn, instances, keypair): """Evacuate instances from failed compute node Scenario: 1. Create net01, net01__subnet: neutron net-create net01 neutron subnet-create net01 192.168.1.0/24 --enable-dhcp \ --name net01__subnet 2. Boot instances vm1 and vm2 in net01 on a single compute node: 3. Destroy a compute node where instances are scheduled 4. Evacuate instances vm1 and vm2: nova evacuate vm1 && nova evacuate vm2 5. Check that they are rescheduled onto another compute node and are in ACTIVE state: 6. Check that pings between vm1 and vm2 are successful """ compute_host = getattr(instances[0], 'OS-EXT-SRV-ATTR:hypervisor_hostname') compute_node = env.find_node_by_fqdn(compute_host) devops_node = devops_env.get_node_by_mac(compute_node.data['mac']) devops_node.destroy() def is_hypervisor_down(): hypervisor = os_conn.nova.hypervisors.find( hypervisor_hostname=compute_host) return hypervisor.state == 'down' common.wait( is_hypervisor_down, timeout_seconds=5 * 60, waiting_for='hypervisor {0} to be in down state'.format(compute_host)) for instance in instances: os_conn.nova.servers.evacuate(instance) def is_instances_migrate(): for instance in os_conn.nova.servers.list(): if instance not in instances: continue if instance.status == 'ERROR': raise Exception('Instance {0.name} is in ERROR status\n' '{0.fault[message]}\n' '{0.fault[details]}'.format(instance)) if not os_conn.server_status_is(instance, 'ACTIVE'): return False if getattr(instance, 'OS-EXT-SRV-ATTR:hypervisor_hostname') == compute_host: return False return True common.wait(is_instances_migrate, timeout_seconds=5 * 60, waiting_for='instances to migrate to another compute') network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair)
def test_cold_migration_for_huge_pages_2m(self, env, os_conn, networks, nfv_flavor, security_group, aggregate): """This test checks that cold migration executed successfully for instances created on computes with huge pages 2M Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Launch instance vm1 in net1 with m1.small.hpgs 3. Check that vm1 is created on compute with huge pages 4. Launch instance vm2 in net2 with m1.small.hpgs 5. Check that vm2 is created on compute with huge pages 6. Check vms connectivity 7. Cold migrate vm1 and check that vm moved to other compute with huge pages 8. Check vms connectivity """ free_pages = {0: 1024, 1: 768, 2: 512} hosts = aggregate.hosts vms = [] vm_hosts = [] for i in range(2): vm = os_conn.create_server(name='vm{}'.format(i), flavor=nfv_flavor[0].id, security_groups=[security_group.id], nics=[{ 'net-id': networks[i] }]) vms.append(vm) for vm in vms: host = getattr(vm, "OS-EXT-SRV-ATTR:host") assert host in hosts vm_hosts.append(host) for host in hosts: self.check_pages(os_conn, host, total_pages=1024, free_pages=free_pages[vm_hosts.count(host)]) for vm in vms: self.check_instance_page_size(os_conn, vm, size=2048) network_checks.check_vm_connectivity(env, os_conn) vm_0_new = self.migrate(os_conn, vms[0]) vm_host_0_new = getattr(vm_0_new, "OS-EXT-SRV-ATTR:host") assert vm_host_0_new in hosts assert vm_host_0_new != vm_hosts.pop(0) vm_hosts.append(vm_host_0_new) for host in hosts: self.check_pages(os_conn, host, total_pages=1024, free_pages=free_pages[vm_hosts.count(host)]) self.check_instance_page_size(os_conn, vm_0_new, size=2048) network_checks.check_vm_connectivity(env, os_conn)
def _prepare_openstack(self): """Prepare OpenStack for scenarios run Steps: 1. Create network1 2. Create router1 and connect it with network1 and external net 3. Boot vm1 in network1 and associate floating ip 4. Boot vm2 in network2 5. Add rules for ping 6. ping 8.8.8.8 from vm2 7. ping vm1 from vm2 and vm1 from vm2 """ # init variables exist_networks = self.os_conn.list_networks()['networks'] ext_network = [x for x in exist_networks if x.get('router:external')][0] self.zone = self.os_conn.nova.availability_zones.find(zoneName="nova") self.hosts = self.zone.hosts.keys() self.instance_keypair = self.os_conn.create_key(key_name='instancekey') self.setup_rules_for_default_sec_group() # create router self.router = self.os_conn.create_router(name="router01")['router'] self.os_conn.router_gateway_add(router_id=self.router['id'], network_id=ext_network['id']) logger.info('router {} was created'.format(self.router['id'])) # create one network by amount of the compute hosts self.net_id = self.os_conn.add_net(self.router['id']) # create two instances in that network # each instance is on the own compute for i, hostname in enumerate(self.hosts, 1): self.os_conn.create_server( name='server%02d' % i, availability_zone='{}:{}'.format(self.zone.zoneName, hostname), key_name=self.instance_keypair.name, nics=[{'net-id': self.net_id}]) # check pings network_checks.check_vm_connectivity(self.env, self.os_conn) # make a list of all ovs agent ids self.ovs_agent_ids = [ agt['id'] for agt in self.os_conn.neutron.list_agents( binary='neutron-openvswitch-agent')['agents']] # make a list of ovs agents that resides only on controllers controllers = [node.data['fqdn'] for node in self.env.get_nodes_by_role('controller')] ovs_agts = self.os_conn.neutron.list_agents( binary='neutron-openvswitch-agent')['agents'] self.ovs_conroller_agents = [agt['id'] for agt in ovs_agts if agt['host'] in controllers]
def test_shutdown_primary_controller_with_l3_agt(self): """[Neutron VLAN and VXLAN] Shut down primary controller and check l3-agent Steps: 1. Check on what agents is router1: neutron l3-agent-list-hosting-router router1 2. If there isn't agent on the primary controller: neutron l3-agent-router-remove non_on_primary_agent_id router1 neutron l3-agent-router-add on_primary_agent_id router1 3. Destroy primary controller virsh destroy <primary_controller> 4. Wait some time until all agents are up neutron-agent-list 5. Check that all routers reschedule from primary controller: neutron router-list-on-l3-agent <on_primary_agent_id> 6. Boot vm3 in network1 7. ping 8.8.8.8 from vm3 8. ping between vm1 and vm3 by internal ip 9. ping between vm1 and vm2 by floating ip """ self._prepare_openstack() # Get current L3 agent on router01 l3_agent = self.os_conn.neutron.list_l3_agent_hosting_routers( self.router['id'])['agents'][0] # Check if the agent is not on the primary controller # Reschedule if needed if l3_agent['host'] != self.primary_host: self.os_conn.reschedule_router_to_primary_host( self.router['id'], self.primary_host) l3_agent = self.os_conn.neutron.list_l3_agent_hosting_routers( self.router['id'])['agents'][0] # virsh destroy of the primary controller self.env.destroy_nodes([self.primary_node]) # Excluding the id of the l3_agent from the list # since it will stay on the destroyed controller # and remain disabled self.l3_agent_ids.remove(l3_agent['id']) # Then check that the rest l3 agents are alive self.os_conn.wait_agents_alive(self.l3_agent_ids) # Check that there are no routers on the first agent self.check_no_routers_on_l3_agent(l3_agent['id']) self.os_conn.add_server(self.networks[0], self.instance_keypair.name, self.hosts[0], self.security_group.id) # Create one more server and check connectivity network_checks.check_vm_connectivity(self.env, self.os_conn)
def test_vms_connectivity_after_evacuation(self, env, os_conn, networks, flavors, aggregate, security_group, devops_env): """This test checks vms connectivity for vms with cpu pinning with 1 NUMA after evacuation Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Boot vm0 with cpu flavor on host0 and net0 3. Boot vm1 with old flavor on host1 and net1 4. Check vms connectivity 5. Kill compute0 and evacuate vm0 to compute1 with --on-shared-storage parameter 6. Check vms connectivity 7. Check numa nodes for vm0 8. Make compute0 alive 9. Check that resources for vm0 were deleted from compute0 """ cpus = get_cpu_distribition_per_numa_node(env) hosts = aggregate.hosts vms = [] for i in range(2): vm = os_conn.create_server( name='vm{}'.format(i), flavor=flavors[i].id, nics=[{'net-id': networks[i]}], availability_zone='nova:{}'.format(hosts[i]), security_groups=[security_group.id]) vms.append(vm) network_checks.check_vm_connectivity(env, os_conn) self.check_cpu_for_vm(os_conn, vms[0], 1, cpus[hosts[0]]) self.compute_change_state(os_conn, devops_env, hosts[0], state='down') vm0_new = self.evacuate(os_conn, devops_env, vms[0]) vm0_new.get() new_host = getattr(vm0_new, "OS-EXT-SRV-ATTR:host") assert new_host in hosts, "Unexpected host after evacuation" assert new_host != hosts[0], "Host didn't change after evacuation" os_conn.wait_servers_ssh_ready(vms) network_checks.check_vm_connectivity(env, os_conn) self.check_cpu_for_vm(os_conn, vm0_new, 1, cpus[new_host]) self.compute_change_state(os_conn, devops_env, hosts[0], state='up') old_hv = os_conn.nova.hypervisors.find(hypervisor_hostname=hosts[0]) assert old_hv.running_vms == 0, ( "Old hypervisor {0} shouldn't have running vms").format(hosts[0]) instance_name = getattr(vm0_new, "OS-EXT-SRV-ATTR:instance_name") assert instance_name in self.get_instances(os_conn, new_host), ( "Instance should be in the list of instances on the new host") assert instance_name not in self.get_instances(os_conn, hosts[0]), ( "Instance shouldn't be in the list of instances on the old host")
def test_ban_l3_agents_many_times(self): """Ban l3-agent many times and check health of l3-agent Scenario: 1. Revert snapshot with neutron cluster 2. Create network1, network2 3. Create router1 and connect it with network1, network2 and external net 4. Boot vm1 in network1 and associate floating ip 5. Boot vm2 in network2 6. Add rules for ping 7. ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other 8. Ban l3-agent on what router1 is 9. Wait for route rescheduling 10. Repeat steps 7-8 11. Ban l3-agent on what router1 is 12. Wait for L3 agent dies 13. Clear last banned L3 agent 14. Wait for L3 agent alive 15. Repeat steps 11-14 40 times 16. Boot one more VM (VM3) in network1 17. Boot vm3 in network1 18. ping 8.8.8.8, vm1 (both ip), vm2 (fixed ip) and vm3 (fixed ip) from each other vm Duration 30m """ net_id = self.os_conn.neutron.list_networks( name="net01")['networks'][0]['id'] devops_node = self.get_node_with_dhcp(net_id) ip = devops_node.data['ip'] # ban 2 l3 agents for _ in range(2): self.ban_l3_agent(router_name="router01", _ip=ip) for i in range(40): # ban l3 agent logger.info('Ban/clear L3 agent. Iteration #{} from 40'.format(i)) last_banned_node = self.ban_l3_agent(router_name="router01", _ip=ip, wait_for_migrate=False, wait_for_die=True) # clear last banned l3 agent self.clear_l3_agent(_ip=ip, router_name="router01", node=last_banned_node, wait_for_alive=True) # check pings network_checks.check_vm_connectivity(self.env, self.os_conn, timeout=10 * 60)
def test_ban_l3_agents_many_times(self): """Ban l3-agent many times and check health of l3-agent Scenario: 1. Revert snapshot with neutron cluster 2. Create network1, network2 3. Create router1 and connect it with network1, network2 and external net 4. Boot vm1 in network1 and associate floating ip 5. Boot vm2 in network2 6. Add rules for ping 7. ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other 8. Ban l3-agent on what router1 is 9. Wait for route rescheduling 10. Repeat steps 7-8 11. Ban l3-agent on what router1 is 12. Wait for L3 agent dies 13. Clear last banned L3 agent 14. Wait for L3 agent alive 15. Repeat steps 11-14 40 times 16. Boot one more VM (VM3) in network1 17. Boot vm3 in network1 18. ping 8.8.8.8, vm1 (both ip), vm2 (fixed ip) and vm3 (fixed ip) from each other vm Duration 30m """ net_id = self.os_conn.neutron.list_networks( name="net01")['networks'][0]['id'] devops_node = self.get_node_with_dhcp(net_id) ip = devops_node.data['ip'] # ban 2 l3 agents for _ in range(2): self.ban_l3_agent(router_name="router01", _ip=ip) for _ in range(40): # ban l3 agent last_banned_node = self.ban_l3_agent(router_name="router01", _ip=ip, wait_for_migrate=False, wait_for_die=True) # clear last banned l3 agent self.clear_l3_agent(_ip=ip, router_name="router01", node=last_banned_node, wait_for_alive=True) # check pings network_checks.check_vm_connectivity(self.env, self.os_conn)
def test_hp_distribution_1g_2m_for_vms(self, env, os_conn, computes_with_mixed_hp, networks, flavors, security_group): """This test checks huge pages 1Gb and 2Mb distribution for vms Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Create vm1 in net1 on compute1 with 1Gb flavor 3. Create vm2 in net2 on compute2 with 2Mb flavor 4. Create vm3 in net2 on compute1 with 2Mb flavor 5. Check instances configuration (about huge pages) 6. Check quantity of HP on computes 7. Check pings from all vms to all vms by all ips """ small_nfv_flavor, medium_nfv_flavor = flavors[0], flavors[1] count_to_allocate_2mb = small_nfv_flavor.ram * 1024 / page_2mb count_to_allocate_1gb = medium_nfv_flavor.ram * 1024 / page_1gb initial_conf = computes_configuration(env) hosts = computes_with_mixed_hp vms_params = [ (hosts[0], networks[0], medium_nfv_flavor, page_1gb), (hosts[1], networks[1], small_nfv_flavor, page_2mb), (hosts[0], networks[1], small_nfv_flavor, page_2mb), ] vms = {} for i, (host, network, flavor, size) in enumerate(vms_params): vm = os_conn.create_server( name='vm{}'.format(i), flavor=flavor.id, nics=[{'net-id': network}], availability_zone='nova:{}'.format(host), security_groups=[security_group.id]) vms.update({vm: size}) for vm, exp_size in vms.items(): assert self.get_instance_page_size(os_conn, vm) == exp_size vms_distribution = [(hosts[0], 1, 1), (hosts[1], 0, 1), ] final_conf = computes_configuration(env) for (host, nr_1gb, nr_2mb) in vms_distribution: exp_free_1g = (initial_conf[host][page_1gb]['total'] - nr_1gb * count_to_allocate_1gb) exp_free_2m = (initial_conf[host][page_2mb]['total'] - nr_2mb * count_to_allocate_2mb) assert exp_free_1g == final_conf[host][page_1gb]['free'] assert exp_free_2m == final_conf[host][page_2mb]['free'] os_conn.wait_servers_ssh_ready(vms.keys()) network_checks.check_vm_connectivity(env, os_conn)
def test_kill_active_l3_agt(self): """[Neutron VLAN and VXLAN] Kill l3-agent process 8. get node with l3 agent where is the router1: neutron l3-agent-hosting-router router1 9. on this node find l3-agent process: ps aux | grep l3-agent 10. Kill it: kill -9 <pid> 11. Wait some time until all agents are up neutron-agent-list 12. Boot vm3 in network1 13. ping 8.8.8.8 from vm3 14. ping between vm1 and vm3 by internal ip 15. ping between vm1 and vm2 by floating ip """ self._prepare_openstack() # Get current L3 agent on router01 router_agt = self.os_conn.neutron.list_l3_agent_hosting_routers( self.router['id'])['agents'][0] # Find the current controller ip with the router01 controller_ip = '' for node in self.env.get_all_nodes(): if node.data['fqdn'] == router_agt['host']: controller_ip = node.data['ip'] break # If ip is empty than no controller with the router was found assert controller_ip, "No controller with the router was found" with self.env.get_ssh_to_node(controller_ip) as remote: cmd = "ps -aux | grep [n]eutron-l3-agent | awk '{print $2}'" result = remote.execute(cmd) pid = result['stdout'][0] logger.info('Got l3 agent pid {}'.format(pid)) logger.info('Now going to kill it on the controller {}'.format( controller_ip)) result = remote.execute('kill -9 {}'.format(pid)) assert result['exit_code'] == 0, "kill failed {}".format(result) self.os_conn.wait_agents_alive(self.l3_agent_ids) # Create one more server and check connectivity self.os_conn.add_server(self.networks[0], self.instance_keypair.name, self.hosts[0], self.security_group.id) network_checks.check_vm_connectivity(self.env, self.os_conn)
def test_huge_pages_distribution(self, env, os_conn, networks, keypair, nfv_flavor, security_group, aggregate): """This test checks huge pages' distribution with flavor for 2M and 1G Huge Pages Steps: 1. Create flavors m1.small.hpgs and m1.small.hpgs-1 2. Create net01 with subnet, net02 with subnet and router01 with interfaces to both nets 3. Launch instance vm1 on compute-1 in net01 with m1.small.hpgs-1 4. Launch instance vm2 on compute-2 in net02 with m1.small.hpgs 5. Launch instance vm3 on compute-1 in net02 with m1.small.hpgs 6. Check vms connectivity """ hosts = aggregate.hosts vm_1 = os_conn.create_server(name='vm1', flavor=nfv_flavor[1].id, key_name=keypair.name, nics=[{ 'net-id': networks[0] }], availability_zone='nova:{}'.format( hosts[0]), security_groups=[security_group.id]) vm_2 = os_conn.create_server(name='vm2', flavor=nfv_flavor[0].id, key_name=keypair.name, nics=[{ 'net-id': networks[1] }], availability_zone='nova:{}'.format( hosts[1]), security_groups=[security_group.id]) vm_3 = os_conn.create_server(name='vm3', flavor=nfv_flavor[0].id, key_name=keypair.name, nics=[{ 'net-id': networks[1] }], availability_zone='nova:{}'.format( hosts[0]), security_groups=[security_group.id]) self.check_pages(os_conn, hosts[0], total_pages=1024, free_pages=512) self.check_pages(os_conn, hosts[1], total_pages=1024, free_pages=512) self.check_instance_page_size(os_conn, vm_1, size=1048576) self.check_instance_page_size(os_conn, vm_2, size=2048) self.check_instance_page_size(os_conn, vm_3, size=2048) network_checks.check_vm_connectivity(env, os_conn)
def test_allocation_huge_pages_2m_for_vms(self, env, os_conn, networks, nfv_flavor, security_group, aggregate): """This test checks allocation 2M HugePages for instances Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Launch vm1 and vm2 using net1 on the first compute 3. Launch vm3 using net1 on the second compute 4. Launch vm4 using net2 on the second compute 5. Check instances configuration (about huge pages) 6. Check quantity of HP on computes 7. Associate floating to vm1 8. Check pings from all vms to all vms by all ips """ free_pages = {1: 768, 3: 256} hosts = aggregate.hosts vms = [] for i in range(2): vm_name = 'vm{}'.format(i) vm = os_conn.create_server( name=vm_name, flavor=nfv_flavor[0].id, nics=[{'net-id': networks[0]}], availability_zone='nova:{}'.format(hosts[0]), security_groups=[security_group.id]) vms.append(vm) vm2 = os_conn.create_server( name='vm2', flavor=nfv_flavor[0].id, nics=[{'net-id': networks[1]}], availability_zone='nova:{}'.format(hosts[0]), security_groups=[security_group.id]) vm3 = os_conn.create_server( name='vm3', flavor=nfv_flavor[0].id, nics=[{'net-id': networks[1]}], availability_zone='nova:{}'.format(hosts[1]), security_groups=[security_group.id]) vms.extend([vm2, vm3]) for vm in vms: self.check_instance_page_size(os_conn, vm, size=2048) self.check_pages(os_conn, hosts[0], total_pages=1024, free_pages=free_pages[3]) self.check_pages(os_conn, hosts[1], total_pages=1024, free_pages=free_pages[1]) os_conn.assign_floating_ip(vms[0]) network_checks.check_vm_connectivity(env, os_conn)
def test_allocation_huge_pages_2m_for_vms(self, env, os_conn, networks, computes_with_hp_2mb, flavors, security_group): """This test checks allocation 2M HugePages for instances Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Launch vm1 and vm2 using net1 on the first compute 3. Launch vm3 using net1 on the second compute 4. Launch vm4 using net2 on the second compute 5. Check instances configuration (about huge pages) 6. Check quantity of HP on computes 7. Associate floating to vm1 8. Check pings from all vms to all vms by all ips """ small_nfv_flavor = flavors[0] count_to_allocate_2mb = small_nfv_flavor.ram * 1024 / page_2mb initial_conf = computes_configuration(env) hosts = computes_with_hp_2mb vms = [] vms_params = [ (hosts[0], networks[0]), (hosts[0], networks[0]), (hosts[0], networks[1]), (hosts[1], networks[1]), ] for i, (host, network) in enumerate(vms_params): vm = os_conn.create_server( name='vm{}'.format(i), flavor=small_nfv_flavor.id, nics=[{'net-id': network}], availability_zone='nova:{}'.format(host), security_groups=[security_group.id]) vms.append(vm) for vm in vms: assert self.get_instance_page_size(os_conn, vm) == page_2mb vms_distribution = [(hosts[0], 3), (hosts[1], 1), ] final_conf = computes_configuration(env) for (host, nr_2mb) in vms_distribution: exp_free_2m = (initial_conf[host][page_2mb]['total'] - nr_2mb * count_to_allocate_2mb) assert exp_free_2m == final_conf[host][page_2mb]['free'] os_conn.assign_floating_ip(vms[0]) os_conn.wait_servers_ssh_ready(vms) network_checks.check_vm_connectivity(env, os_conn)
def test_cold_migration_for_huge_pages_2m( self, env, os_conn, networks, nfv_flavor, security_group, aggregate): """This test checks that cold migration executed successfully for instances created on computes with huge pages 2M Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Launch instance vm1 in net1 with m1.small.hpgs 3. Check that vm1 is created on compute with huge pages 4. Launch instance vm2 in net2 with m1.small.hpgs 5. Check that vm2 is created on compute with huge pages 6. Check vms connectivity 7. Cold migrate vm1 and check that vm moved to other compute with huge pages 8. Check vms connectivity """ free_pages = {0: 1024, 1: 768, 2: 512} hosts = aggregate.hosts vms = [] vm_hosts = [] for i in range(2): vm = os_conn.create_server( name='vm{}'.format(i), flavor=nfv_flavor[0].id, security_groups=[security_group.id], nics=[{'net-id': networks[i]}]) vms.append(vm) for vm in vms: host = getattr(vm, "OS-EXT-SRV-ATTR:host") assert host in hosts vm_hosts.append(host) for host in hosts: self.check_pages(os_conn, host, total_pages=1024, free_pages=free_pages[vm_hosts.count(host)]) for vm in vms: self.check_instance_page_size(os_conn, vm, size=2048) network_checks.check_vm_connectivity(env, os_conn) vm_0_new = self.migrate(os_conn, vms[0]) vm_host_0_new = getattr(vm_0_new, "OS-EXT-SRV-ATTR:host") assert vm_host_0_new in hosts assert vm_host_0_new != vm_hosts.pop(0) vm_hosts.append(vm_host_0_new) for host in hosts: self.check_pages(os_conn, host, total_pages=1024, free_pages=free_pages[vm_hosts.count(host)]) self.check_instance_page_size(os_conn, vm_0_new, size=2048) network_checks.check_vm_connectivity(env, os_conn)
def prepare_openstack(self, init): """Prepare OpenStack for scenarios run Steps: 1. Create network1, network2 2. Create router1 and connect it with network1, network2 and external net 3. Boot vm1 in network1 and associate floating ip 4. Boot vm2 in network2 5. Add rules for ping 6. Ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other """ # init variables self.zone = self.os_conn.nova.availability_zones.find(zoneName="nova") self.security_group = self.os_conn.create_sec_group_for_ssh() self.hosts = self.zone.hosts.keys()[:2] self.instance_keypair = self.os_conn.create_key(key_name='instancekey') # create router router = self.os_conn.create_router(name="router01") self.os_conn.router_gateway_add( router_id=router['router']['id'], network_id=self.os_conn.ext_network['id']) # create 2 networks and 2 instances for i, hostname in enumerate(self.hosts, 1): network = self.os_conn.create_network(name='net%02d' % i) subnet = self.os_conn.create_subnet( network_id=network['network']['id'], name='net%02d__subnet' % i, cidr="192.168.%d.0/24" % i) self.os_conn.router_interface_add( router_id=router['router']['id'], subnet_id=subnet['subnet']['id']) self.os_conn.create_server( name='server%02d' % i, availability_zone='{}:{}'.format(self.zone.zoneName, hostname), key_name=self.instance_keypair.name, nics=[{'net-id': network['network']['id']}], security_groups=[self.security_group.id]) # add floating ip to first server server1 = self.os_conn.nova.servers.find(name="server01") self.os_conn.assign_floating_ip(server1) # check pings network_checks.check_vm_connectivity(self.env, self.os_conn)
def test_ban_one_l3_agent(self, ban_count): """Check l3-agent rescheduling after l3-agent dies on vlan Scenario: 1. Revert snapshot with neutron cluster 2. Create network1, network2 3. Create router1 and connect it with network1, network2 and external net 4. Boot vm1 in network1 and associate floating ip 5. Boot vm2 in network2 6. Add rules for ping 7. ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other 8. get node with l3 agent on what is router1 9. ban this l3 agent on the node with pcs (e.g. pcs resource ban neutron-l3-agent node-3.test.domain.local) 10. wait some time (about 20-30) while pcs resource and neutron agent-list will show that it is dead 11. Check that router1 was rescheduled 12. Boot vm3 in network1 13. ping 8.8.8.8, vm1 (both ip), vm2 (fixed ip) and vm3 (fixed ip) from each other Duration 10m """ net_id = self.os_conn.neutron.list_networks( name="net01")['networks'][0]['id'] devops_node = self.get_node_with_dhcp(net_id) ip = devops_node.data['ip'] # ban l3 agent for _ in range(ban_count): self.ban_l3_agent(_ip=ip, router_name="router01") # create another server on net01 net01 = self.os_conn.nova.networks.find(label="net01") self.os_conn.create_server( name='server03', availability_zone='{}:{}'.format(self.zone.zoneName, self.hosts[0]), key_name=self.instance_keypair.name, nics=[{'net-id': net01.id}], security_groups=[self.security_group.id]) # check pings network_checks.check_vm_connectivity(self.env, self.os_conn)
def test_restart_primary_controller_with_l3_agt(self): """[Neutron VLAN and VXLAN] Reset primary controller and check l3-agent Steps: 1. Check on what agents is router1: neutron l3-agent-list-hosting-router router1 2. If there isn't agent on the primary controller: neutron l3-agent-router-remove non_on_primary_agent_id router1 neutron l3-agent-router-add on_primary_agent_id router1 3. Restart primary controller 4. Wait some time until all agents are up neutron-agent-list 5. Check that all routers reschedule from primary controller: neutron router-list-on-l3-agent <on_primary_agent_id> 6. Boot vm3 in network1 7. ping 8.8.8.8 from vm3 8. ping between vm1 and vm3 by internal ip 9. ping between vm1 and vm2 by floating ip """ self._prepare_openstack() # Get current L3 agent on router01 router_agt = self.os_conn.neutron.list_l3_agent_hosting_routers( self.router['id'])['agents'][0] # Check if the agent is not on the primary controller # Reschedule if needed if router_agt['host'] != self.primary_host: self.os_conn.reschedule_router_to_primary_host(self.router['id'], self.primary_host) router_agt = self.os_conn.neutron.list_l3_agent_hosting_routers( self.router['id'])['agents'][0] # virsh destroy of the primary controller self.env.warm_restart_nodes([self.primary_node]) # Check that the all l3 are alive self.os_conn.wait_agents_alive(self.l3_agent_ids) # Check that there are no routers on the first agent self.check_no_routers_on_l3_agent(router_agt['id']) # Create one more server and check connectivity self.os_conn.add_server(self.networks[0], self.instance_keypair.name, self.hosts[0], self.security_group.id) network_checks.check_vm_connectivity(self.env, self.os_conn)
def test_vms_connectivity_after_evacuation(self, env, os_conn, volume, computes_with_dpdk_hp, flavors, networks, keypair, devops_env, security_group): """This test checks connectivity between VMs with DPDK after evacuation. Please note we're not able to count DPDK huge pages only, they're added to count of 2Mb huge pages. Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Create flavor for huge pages with 512Mb ram, 1 vcpu and 1Gb disk 3. Launch vm1 (from not empty volume), vm2 on compute-1, vm3 - on compute-2, vm1 in net1, vm2 and vm3 in net2 4. Kill compute2 and evacuate vm3 5. Check vms connectivity 6. Start compute2 7. Check instance page size 8. Check that neutron port has binding:vif_type = vhostuser 9. Check that count of free 2Mb huge pages is expected one for each host """ hosts = computes_with_dpdk_hp initial_conf = computes_configuration(env) vms_param = [(hosts[0], networks[0], {'vda': volume.id}), (hosts[0], networks[1], None), (hosts[1], networks[1], None)] vms = self.create_vms(os_conn, hosts, networks, flavors[0], keypair, security_group, vms_param=vms_param) network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair) self.compute_change_state(os_conn, devops_env, hosts[1], state='down') vm_new = self.evacuate(os_conn, devops_env, vms[2], on_shared_storage=False) vm_new_host = getattr(os_conn.nova.servers.get(vm_new), "OS-EXT-SRV-ATTR:host") assert vm_new_host in hosts assert vm_new_host != hosts[1] network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair) self.compute_change_state(os_conn, devops_env, hosts[1], state='up') final_conf = computes_configuration(env) exp_hosts_usage = [(hosts[0], 3), (hosts[1], 0)] for (host, nr_2mb) in exp_hosts_usage: exp_free_2m = (initial_conf[host][page_2mb]['free'] - nr_2mb * flavors[0].ram * 1024 / page_2mb) assert exp_free_2m == final_conf[host][page_2mb]['free']
def test_allocation_huge_pages_2m_for_vms_with_old_flavor( self, env, os_conn, networks, computes_with_hp_2mb, flavors, computes_without_hp, security_group): """This test checks that Huge pages set for vm1, vm2 and vm3 shouldn't use Huge pages, connectivity works properly Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Create vm1 in net1 on compute1 with 2Mb flavor 3. Create vm2 in net2 on compute2 with old flavor 4. Create vm3 in net1 on compute1 with old flavor 5. Check huge pages. Check that it was allocated only HP for vm1 6. Check pings from all vms to all vms by all ips """ small_nfv_flavor, old_flavor = flavors[0], flavors[2] count_to_allocate_2mb = small_nfv_flavor.ram * 1024 / page_2mb initial_conf = computes_configuration(env) hosts_hp = computes_with_hp_2mb hosts_no_hp = computes_without_hp vms_params = [ (hosts_hp[0], networks[0], small_nfv_flavor, page_2mb), (hosts_no_hp[0], networks[1], old_flavor, None), (hosts_hp[0], networks[0], old_flavor, None)] vms = {} for i, (host, network, flavor, size) in enumerate(vms_params): vm = os_conn.create_server( name='vm{}'.format(i), flavor=flavor.id, nics=[{'net-id': network}], availability_zone='nova:{}'.format(host), security_groups=[security_group.id]) vms.update({vm: size}) for vm, exp_size in vms.items(): assert self.get_instance_page_size(os_conn, vm) == exp_size vms_distribution = [(hosts_hp[0], 1), (hosts_no_hp[0], 0), ] final_conf = computes_configuration(env) for (host, nr_2mb) in vms_distribution: exp_free_2m = (initial_conf[host][page_2mb]['total'] - nr_2mb * count_to_allocate_2mb) assert exp_free_2m == final_conf[host][page_2mb]['free'] os_conn.wait_servers_ssh_ready(vms.keys()) network_checks.check_vm_connectivity(env, os_conn)
def test_resizing_of_vms_with_huge_pages(self, env, os_conn, computes_with_mixed_hp, networks, flavors, security_group): """This test checks resizing of VM with flavor for 2M to flavor for 1G flavor and on old flavor Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Create vm1 in net1 on compute1 with 2Mb flavor 3. Create vm2 in net2 on compute2 with old flavor 4. Check instances configuration (about huge pages) 5. Check pings from all vms to all vms by all ips 6. Resize vm1 to 1Gb and check ping 7. Resize vm1 to old and check ping 8. Resize vm1 to 1Gb and check ping 9. Resize vm1 to 2Mb and check ping """ small_nfv_flv, meduim_nfv_flv, old_flv = flavors hosts = computes_with_mixed_hp vms_params = [ (hosts[0], networks[0], small_nfv_flv, page_2mb), (hosts[1], networks[1], old_flv, None), ] vms = {} for i, (host, network, flavor, size) in enumerate(vms_params): vm = os_conn.create_server( name='vm{}'.format(i), flavor=flavor.id, nics=[{'net-id': network}], availability_zone='nova:{}'.format(host), security_groups=[security_group.id]) vms.update({vm: size}) for vm, exp_size in vms.items(): assert self.get_instance_page_size(os_conn, vm) == exp_size params = [(meduim_nfv_flv, page_1gb), (old_flv, None), (meduim_nfv_flv, page_1gb), (small_nfv_flv, page_2mb), ] for (flavor, size) in params: self.resize(os_conn, vms.keys()[0], flavor_to_resize=flavor) assert self.get_instance_page_size(os_conn, vms.keys()[0]) == size os_conn.wait_servers_ssh_ready(vms.keys()) network_checks.check_vm_connectivity(env, os_conn)
def test_VLAN_Allo_2M_HP_vms_req_HP_vms_with_old_flavor(self, env, os_conn, networks, nfv_flavor, keypair, security_group, volume, aggregate): """ This test checks that Huge pages set for vm1, vm2 and vm3 shouldn't use Huge pages, connectivity works properly Steps: 1. Create net01, subnet. 2. Create net 02, subnet. 3. Create router, set gateway and add interfaces to both networks 4. Launch vms using m1.small.hpgs flavor: vm1, vm3 on compute-1, vm2 on compute-2. For vm1 use new flavor, for vm2 and vm3 - old 5. Locate the part of all instances configuration that is relevant to huge pages: #on controller hypervisor=nova show hpgs-test | grep OS-EXT-SRV-ATTR:host | cut -d\| -f3 instance=nova show hpgs-test | grep OS-EXT-SRV-ATTR:instance_name | cut -d\| -f3 # on compute virsh dumpxml $instance |awk '/memoryBacking/ {p=1}; p; /\/numatune/ {p=0}' 6. ping vm2 from vm1 7. ping vm2 from vm3 8. ping vm3 from vm1 9. Check that it was allocated only HP for vm1 """ hosts = aggregate.hosts vm_0 = os_conn.create_server( name='vm1', flavor=nfv_flavor[0].id, key_name=keypair.name, nics=[{'net-id': networks[0]}], availability_zone='nova:{}'.format(hosts[0]), security_groups=[security_group.id]) vm_1 = os_conn.create_server( name='vm2', flavor=nfv_flavor[0].id, key_name=keypair.name, availability_zone='nova:{}'.format(hosts[1]), security_groups=[security_group.id], nics=[{'net-id': networks[1]}]) vm_2 = os_conn.create_server( name='vm3', flavor=nfv_flavor[0].id, key_name=keypair.name, nics=[{'net-id': networks[1]}], availability_zone='nova:{}'.format(hosts[1]), security_groups=[security_group.id]) vms = [vm_0, vm_1, vm_2] self.check_pages(os_conn, hosts[0], total_pages=1024, free_pages=768) self.check_pages(os_conn, hosts[1], total_pages=1024, free_pages=512) for vm in vms: self.check_instance_page_size(os_conn, vm, size=2048) network_checks.check_vm_connectivity(env, os_conn)
def test_cpu_pinning_migration( self, env, os_conn, networks, flavors, security_group, aggregate): """This test checks that cpu pinning executed successfully for instances created on computes with 1 NUMA Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Launch vm1 using m1.small.performance flavor on compute-1 and vm2 on compute-2. 3. Migrate vm1 from compute-1 4. Check CPU Pinning """ hosts = aggregate.hosts vms = [] cpus = get_cpu_distribition_per_numa_node(env) for i in range(2): vms.append(os_conn.create_server( name='vm{}'.format(i), flavor=flavors[0].id, nics=[{'net-id': networks[0]}], availability_zone='nova:{}'.format(hosts[i]), security_groups=[security_group.id])) for i in range(5): vm_host = getattr(vms[0], "OS-EXT-SRV-ATTR:host") vm_0_new = self.migrate(os_conn, vms[0]) vm_host_0_new = getattr(vm_0_new, "OS-EXT-SRV-ATTR:host") assert vm_host_0_new != vm_host for vm in vms: host = getattr(vm, "OS-EXT-SRV-ATTR:host") self.check_cpu_for_vm(os_conn, os_conn.get_instance_detail(vm), 2, cpus[host]) os_conn.wait_servers_ssh_ready(vms) network_checks.check_vm_connectivity(env, os_conn)
def test_base_vms_connectivity(self, env, os_conn, computes_with_dpdk_hp, networks, keypair, flavors, security_group): """This test checks base connectivity between VMs with DPDK. Please note we're not able to count DPDK huge pages only, they're added to count of 2Mb huge pages. Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Create flavor for huge pages with 512Mb ram, 1 vcpu and 1Gb disk 3. Launch vm1, vm2, vm3 on compute-1 and vm4 on compute-2, vm1 and vm2 in net1, vm3 and vm4 in net2 4. Check vms connectivity 5. Check instance page size 6. Check that neutron port has binding:vif_type = vhostuser 7. Check that count of 2Mb huge pages is expected for each host """ hosts = computes_with_dpdk_hp initial_conf = computes_configuration(env) vms_param = [(hosts[0], networks[0], None), (hosts[0], networks[0], None), (hosts[0], networks[1], None), (hosts[1], networks[1], None)] vms = self.create_vms(os_conn, hosts, networks, flavors[0], keypair, security_group, vms_param) network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair) for vm in vms: self.check_vif_type_for_vm(vm, os_conn) act_size = self.get_instance_page_size(os_conn, env) assert act_size == page_2mb, ( "Unexpected package size. Should be {0} instead of {1}".format( page_2mb, act_size)) final_conf = computes_configuration(env) exp_hosts_usage = [(hosts[0], 3), (hosts[1], 1)] for (host, nr_2mb) in exp_hosts_usage: exp_free_2m = (initial_conf[host][page_2mb]['free'] - nr_2mb * flavors[0].ram * 1024 / page_2mb) assert exp_free_2m == final_conf[host][page_2mb]['free']
def test_huge_pages_distribution( self, env, os_conn, networks, keypair, nfv_flavor, security_group, aggregate): """This test checks huge pages' distribution with flavor for 2M and 1G Huge Pages Steps: 1. Create flavors m1.small.hpgs and m1.small.hpgs-1 2. Create net01 with subnet, net02 with subnet and router01 with interfaces to both nets 3. Launch instance vm1 on compute-1 in net01 with m1.small.hpgs-1 4. Launch instance vm2 on compute-2 in net02 with m1.small.hpgs 5. Launch instance vm3 on compute-1 in net02 with m1.small.hpgs 6. Check vms connectivity """ hosts = aggregate.hosts vm_1 = os_conn.create_server( name='vm1', flavor=nfv_flavor[1].id, key_name=keypair.name, nics=[{'net-id': networks[0]}], availability_zone='nova:{}'.format(hosts[0]), security_groups=[security_group.id]) vm_2 = os_conn.create_server( name='vm2', flavor=nfv_flavor[0].id, key_name=keypair.name, nics=[{'net-id': networks[1]}], availability_zone='nova:{}'.format(hosts[1]), security_groups=[security_group.id]) vm_3 = os_conn.create_server( name='vm3', flavor=nfv_flavor[0].id, key_name=keypair.name, nics=[{'net-id': networks[1]}], availability_zone='nova:{}'.format(hosts[0]), security_groups=[security_group.id]) self.check_pages(os_conn, hosts[0], total_pages=1024, free_pages=512) self.check_pages(os_conn, hosts[1], total_pages=1024, free_pages=512) self.check_instance_page_size(os_conn, vm_1, size=1048576) self.check_instance_page_size(os_conn, vm_2, size=2048) self.check_instance_page_size(os_conn, vm_3, size=2048) network_checks.check_vm_connectivity(env, os_conn)
def test_ssh_connection_after_ovs_restart(self, env, os_conn, computes_with_dpdk_hp, flavors, networks, security_group, keypair, restart_point): """This test checks ssh connection between VMs with DPDK after ovs restart on computes/controllers. Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Create flavor for huge pages with 512Mb ram, 1 vcpu and 1Gb disk 3. Launch vm1, vm2, vm3 on compute-1 and vm4 on compute-2, vm1 and vm2 in net1, vm3 and vm4 in net2 4. Check that neutron port has binding:vif_type = vhostuser 5. Check instance page size 6. Open ssh connection to vm1 and vm4 7. Restart ovs on computes/controllers 8. Check that both ssh connections are still alive 9. Check vms connectivity """ hosts = computes_with_dpdk_hp vms_param = [(hosts[0], networks[0], None), (hosts[0], networks[0], None), (hosts[0], networks[1], None), (hosts[1], networks[1], None)] vms = self.create_vms(os_conn, hosts, networks, flavors[0], keypair, security_group, vms_param) vm1_remote = os_conn.ssh_to_instance(env, vms[0], keypair) vm4_remote = os_conn.ssh_to_instance(env, vms[3], keypair) with vm1_remote, vm4_remote: if restart_point == 'computes': self.restart_ovs_on_computes(env, os_conn) elif restart_point == 'controllers': self.restart_ovs_on_controllers(env, os_conn) vm1_remote.check_call("uname") vm4_remote.check_call("uname") network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair)
def test_vms_connectivity_after_cold_migration(self, env, os_conn, computes_with_dpdk_hp, flavors, networks, keypair, security_group): """This test checks connectivity between VMs with DPDK after cold migration. Please note we're not able to count DPDK huge pages only, they're added to count of 2Mb huge pages. Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Create flavor for huge pages with 512Mb ram, 1 vcpu and 1Gb disk 3. Launch vm1, vm2 on compute-1, vm3 - on compute-2, vm1 in net1, vm2 and vm3 in net2 4. Migrate vm1 and check that vm moved to other compute with huge pages 5. Check instance page size 6. Check that neutron port has binding:vif_type = vhostuser 7. Check vms connectivity """ hosts = computes_with_dpdk_hp vms_param = [(hosts[0], networks[0], None), (hosts[0], networks[1], None), (hosts[1], networks[1], None)] vms = self.create_vms(os_conn, hosts, networks, flavors[0], keypair, security_group, vms_param=vms_param) network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair) vm0_new = self.migrate(os_conn, vms[0]) vm0_host = getattr(os_conn.nova.servers.get(vm0_new), "OS-EXT-SRV-ATTR:host") assert vm0_host in hosts, ("Unexpected host {0}," "should be in {1}".format(vm0_host, hosts)) assert vm0_host != hosts[0], ("New host is expected instead of {0}" "after cold migration".format(hosts[0])) network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair)
def test_vms_connectivity_after_live_migration(self, env, os_conn, computes_with_dpdk_hp, flavors, networks, keypair, security_group): """This test checks connectivity between VMs with DPDK after live migration. Please note we're not able to count DPDK huge pages only, they're added to count of 2Mb huge pages. Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Create flavor for huge pages with 512Mb ram, 1 vcpu and 1Gb disk 3. Launch vm1, vm2 on compute-1, vm3 - on compute-2, vm1 in net1, vm2 and vm3 in net2 4. Live migrate vm1 to compute2 5. Check instance page size 6. Check that neutron port has binding:vif_type = vhostuser 7. Check that count of free 2Mb huge pages is expected one for each host 8. Check vms connectivity """ hosts = computes_with_dpdk_hp initial_conf = computes_configuration(env) vms_param = [(hosts[0], networks[0], None), (hosts[0], networks[1], None), (hosts[1], networks[1], None)] vms = self.create_vms(os_conn, hosts, networks, flavors[0], keypair, security_group, vms_param=vms_param) self.live_migrate(os_conn, vms[0], hosts[1]) network_checks.check_vm_connectivity(env, os_conn, vm_keypair=keypair) final_conf = computes_configuration(env) exp_hosts_usage = [(hosts[0], 1), (hosts[1], 2)] for (host, nr_2mb) in exp_hosts_usage: exp_free_2m = (initial_conf[host][page_2mb]['free'] - nr_2mb * flavors[0].ram * 1024 / page_2mb) assert exp_free_2m == final_conf[host][page_2mb]['free']
def _prepare_openstack(self): """Prepare OpenStack for scenarios run Steps: 1. Create network1, network2 2. Create router1 and connect it with network1, network2 and external net 3. Boot vm1 in network1 and associate floating ip 4. Boot vm2 in network2 5. Add rules for ping 6. ping 8.8.8.8 from vm2 7. ping vm1 from vm2 and vm1 from vm2 """ # init variables exist_networks = self.os_conn.list_networks()['networks'] ext_network = [x for x in exist_networks if x.get('router:external')][0] self.zone = self.os_conn.nova.availability_zones.find(zoneName="nova") self.hosts = self.zone.hosts.keys()[:2] self.instance_keypair = self.os_conn.create_key(key_name='instancekey') self.security_group = self.os_conn.create_sec_group_for_ssh() self.networks = [] # create router self.router = self.os_conn.create_router(name="router01")['router'] self.os_conn.router_gateway_add(router_id=self.router['id'], network_id=ext_network['id']) logger.info( 'router {name}({id}) was created'.format(**self.router)) # create networks by amount of the compute hosts for hostname in self.hosts: net_id = self.os_conn.add_net(self.router['id']) self.networks.append(net_id) self.os_conn.add_server(net_id, self.instance_keypair.name, hostname, self.security_group.id) # add floating ip to first server self.server1 = self.os_conn.nova.servers.find(name="server01") self.os_conn.assign_floating_ip(self.server1) # check pings network_checks.check_vm_connectivity(self.env, self.os_conn) # Find a primary controller primary_controller = self.env.primary_controller self.primary_node = self.devops_env.get_node_by_fuel_node( primary_controller) self.primary_host = primary_controller.data['fqdn'] # Find a non-primary controller non_primary_controller = self.env.non_primary_controllers[0] self.non_primary_node = self.devops_env.get_node_by_fuel_node( non_primary_controller) self.non_primary_host = non_primary_controller.data['fqdn'] # make a list of all l3 agent ids self.l3_agent_ids = [agt['id'] for agt in self.os_conn.neutron.list_agents( binary='neutron-l3-agent')['agents']] self.dhcp_agent_ids = [agt['id'] for agt in self.os_conn.neutron.list_agents( binary='neutron-dhcp-agent')['agents']]
def test_lm_ceph_for_huge_pages(self, env, os_conn, networks, volume, keypair, nfv_flavor, security_group, aggregate): """This test checks that live migration executed successfully for instances created on computes with ceph and huge pages Steps: 1. Create net1 with subnet, net2 with subnet and router1 with interfaces to both nets 2. Launch instance vm1 with volume vol1 on compute-1 in net1 with m1.small.hpgs 3. Launch instance vm2 on compute-2 in net2 with m1.small.hpgs 4. Make volume from vm2 volume_vm 5. Launch instance vm3 on compute-2 in net2 with volume_vm with m1.small.hpgs 6. Check vms connectivity 7. Live migrate vm1 on compute-2 and check that vm moved to compute-2 with Active state 8. Check vms connectivity 9. Live migrate vm2 with block-migrate parameter on compute-1 and check that vm moved to compute-2 with Active state 10. Check vms connectivity 11. Live migrate vm3 on compute-1 and check that vm moved to compute-1 with Active state 12. Check vms connectivity """ hosts = aggregate.hosts vm_0 = os_conn.create_server(name='vm1', flavor=nfv_flavor[0].id, key_name=keypair.name, nics=[{ 'net-id': networks[0] }], availability_zone='nova:{}'.format( hosts[0]), security_groups=[security_group.id], block_device_mapping={'vda': volume.id}) vm_1 = os_conn.create_server(name='vm2', flavor=nfv_flavor[0].id, key_name=keypair.name, availability_zone='nova:{}'.format( hosts[1]), security_groups=[security_group.id], nics=[{ 'net-id': networks[1] }]) volume_vm = self.create_volume_from_vm(os_conn, vm_1) vm_2 = os_conn.create_server(name='vm3', flavor=nfv_flavor[0].id, key_name=keypair.name, nics=[{ 'net-id': networks[1] }], availability_zone='nova:{}'.format( hosts[1]), security_groups=[security_group.id], block_device_mapping={'vda': volume_vm}) vms = [vm_0, vm_1, vm_2] self.check_pages(os_conn, hosts[0], total_pages=1024, free_pages=768) self.check_pages(os_conn, hosts[1], total_pages=1024, free_pages=512) for vm in vms: self.check_instance_page_size(os_conn, vm, size=2048) network_checks.check_vm_connectivity(env, os_conn) self.live_migrate(os_conn, vms[0], hosts[1], block_migration=False) self.check_pages(os_conn, hosts[0], total_pages=1024, free_pages=1024) self.check_pages(os_conn, hosts[1], total_pages=1024, free_pages=256) network_checks.check_vm_connectivity(env, os_conn) self.live_migrate(os_conn, vms[1], hosts[0]) self.check_pages(os_conn, hosts[0], total_pages=1024, free_pages=768) self.check_pages(os_conn, hosts[1], total_pages=1024, free_pages=512) network_checks.check_vm_connectivity(env, os_conn) self.live_migrate(os_conn, vms[2], hosts[0], block_migration=False) self.check_pages(os_conn, hosts[0], total_pages=1024, free_pages=512) self.check_pages(os_conn, hosts[1], total_pages=1024, free_pages=768) for vm in vms: self.check_instance_page_size(os_conn, vm, size=2048) network_checks.check_vm_connectivity(env, os_conn)
def test_shutdown_not_primary_controller(self, env_name): """Shut down non-primary controller and check l3-agent work Scenario: 1. Revert snapshot with neutron cluster 2. Create network1, network2 3. Create router1 and connect it with network1, network2 and external net 4. Boot vm1 in network1 and associate floating ip 5. Boot vm2 in network2 6. Add rules for ping 7. ping 8.8.8.8, vm1 (both ip) and vm2 (fixed ip) from each other 8. Check on what agents is router1 9. If agent on primary controller move it to any other controller 10. Destroy non primary controller 11. Wait for L3 agent dies 12. Check that all routers reschedule from non primary controller 13. Boot one more VM (VM3) in network1 14. Boot vm3 in network1 15. ping 8.8.8.8, vm1 (both ip), vm2 (fixed ip) and vm3 (fixed ip) from each other vm Duration 10m """ router = self.os_conn.neutron.list_routers( name='router01')['routers'][0] l3_agent = self.os_conn.get_l3_for_router(router['id'])['agents'][0] leader_node = self.env.leader_controller # Move router to slave l3 agent, if needed if leader_node.data['fqdn'] == l3_agent['host']: l3_agents = self.os_conn.list_l3_agents() leader_l3_agent = [x for x in l3_agents if x['host'] == leader_node.data['fqdn']][0] self.os_conn.neutron.remove_router_from_l3_agent( leader_l3_agent['id'], router_id=router['id']) slave_l3_agents = [x for x in l3_agents if x != leader_l3_agent] l3_agent = slave_l3_agents[0] self.os_conn.neutron.add_router_to_l3_agent( l3_agent['id'], body={'router_id': router['id']}) # Destroy node with l3 agent node = self.env.find_node_by_fqdn(l3_agent['host']) devops_node = DevopsClient.get_node_by_mac(env_name=env_name, mac=node.data['mac']) if devops_node is not None: devops_node.destroy() else: raise Exception("Can't find devops controller node to destroy it") # Wait for l3 agent die wait( lambda: self.os_conn.get_l3_for_router( router['id'])['agents'][0]['alive'] is False, expected_exceptions=NeutronClientException, timeout_seconds=60 * 5, sleep_seconds=(1, 60, 5), waiting_for="L3 agent is died") # Wait for migrating all routers from died L3 agent wait( lambda: len(self.os_conn.neutron.list_routers_on_l3_agent( l3_agent['id'])['routers']) == 0, timeout_seconds=60 * 5, sleep_seconds=(1, 60, 5), waiting_for="migrating all routers from died L3 agent" ) # create another server on net01 net01 = self.os_conn.nova.networks.find(label="net01") self.os_conn.create_server( name='server03', availability_zone='{}:{}'.format(self.zone.zoneName, self.hosts[0]), key_name=self.instance_keypair.name, nics=[{'net-id': net01.id}], security_groups=[self.security_group.id]) # check pings network_checks.check_vm_connectivity(self.env, self.os_conn)
def _prepare_openstack(self): """Prepare OpenStack for scenarios run Steps: 1. Create network1, network2 2. Create router1 and connect it with network1, network2 and external net 3. Boot vm1 in network1 and associate floating ip 4. Boot vm2 in network2 5. Add rules for ping 6. ping 8.8.8.8 from vm2 7. ping vm1 from vm2 and vm1 from vm2 """ # init variables exist_networks = self.os_conn.list_networks()['networks'] ext_network = [x for x in exist_networks if x.get('router:external')][0] self.zone = self.os_conn.nova.availability_zones.find(zoneName="nova") self.hosts = self.zone.hosts.keys()[:2] self.instance_keypair = self.os_conn.create_key(key_name='instancekey') self.security_group = self.os_conn.create_sec_group_for_ssh() self.networks = [] # create router self.router = self.os_conn.create_router(name="router01")['router'] self.os_conn.router_gateway_add(router_id=self.router['id'], network_id=ext_network['id']) logger.info('router {name}({id}) was created'.format(**self.router)) # create networks by amount of the compute hosts for hostname in self.hosts: net_id = self.os_conn.add_net(self.router['id']) self.networks.append(net_id) self.os_conn.add_server(net_id, self.instance_keypair.name, hostname, self.security_group.id) # add floating ip to first server self.server1 = self.os_conn.nova.servers.find(name="server01") self.os_conn.assign_floating_ip(self.server1) # check pings network_checks.check_vm_connectivity(self.env, self.os_conn) # Find a primary controller primary_controller = self.env.primary_controller mac = primary_controller.data['mac'] self.primary_node = DevopsClient.get_node_by_mac( env_name=self.env_name, mac=mac) self.primary_host = primary_controller.data['fqdn'] # Find a non-primary controller non_primary_controller = self.env.non_primary_controllers[0] mac = non_primary_controller.data['mac'] self.non_primary_node = DevopsClient.get_node_by_mac( env_name=self.env_name, mac=mac) self.non_primary_host = non_primary_controller.data['fqdn'] # make a list of all l3 agent ids self.l3_agent_ids = [ agt['id'] for agt in self.os_conn.neutron.list_agents( binary='neutron-l3-agent')['agents'] ] self.dhcp_agent_ids = [ agt['id'] for agt in self.os_conn.neutron.list_agents( binary='neutron-dhcp-agent')['agents'] ]
def test_restart_openvswitch_agent_under_bat(self): """Restart openvswitch-agents with broadcast traffic background Steps: 1. Go to vm1's console and run arping to initiate broadcast traffic: sudo arping -I eth0 <vm2_fixed_ip> 2. Disable ovs-agents on all controllers 3. Restart service 'neutron-plugin-openvswitch-agent' on all computes 4. Enable ovs-agents back. 5. Check that pings between vm1 and vm2 aren't interrupted or not more than 2 packets are lost """ self._prepare_openstack() # Run arping in background on server01 towards server02 srv_list = self.os_conn.nova.servers.list() srv1 = srv_list.pop() srv2 = srv_list.pop() vm_ip = self.os_conn.get_nova_instance_ips( self.os_conn.nova.servers.find(name=srv2.name))['fixed'] arping_cmd = 'sudo arping -I eth0 {}'.format(vm_ip) cmd = ' '.join((arping_cmd, '< /dev/null > ~/arp.log 2>&1 &')) result = network_checks.run_on_vm(self.env, self.os_conn, srv1, self.instance_keypair, cmd) err_msg = 'Failed to start the arping on vm result: {}'.format(result) assert not result['exit_code'], err_msg # Then check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) # Disable ovs agent on all controllers self.disable_ovs_agents_on_controller() # Then check that all ovs went down self.os_conn.wait_agents_down(self.ovs_conroller_agents) # Restart ovs agent service on all computes self.restart_ovs_agents_on_computes() # Enable ovs agent on all controllers self.enable_ovs_agents_on_controllers() # Then check that all ovs agents are alive self.os_conn.wait_agents_alive(self.ovs_agent_ids) # Check that arping is still executing cmd = 'ps' result = network_checks.run_on_vm(self.env, self.os_conn, srv1, self.instance_keypair, cmd) arping_is_run = False for line in result['stdout']: if arping_cmd in line: arping_is_run = True break err_msg = 'arping was not found in stdout: {}'.format(result['stdout']) assert arping_is_run, err_msg # Read log of arpping execution for future possible debug cmd = 'cat ~/arp.log' result = network_checks.run_on_vm(self.env, self.os_conn, srv1, self.instance_keypair, cmd) logger.debug(result) # Check connectivity network_checks.check_vm_connectivity(self.env, self.os_conn)