def launch_iperf_server(self, vm, keypair, vm_login, vm_pwd):
"""Launch iperf server"""
server_cmd = 'iperf -u -s -p 5002 </dev/null > ~/iperf.log 2>&1 &'
res_srv = network_checks.run_on_vm(self.env, self.os_conn, vm, keypair,
server_cmd, vm_login=vm_login,
vm_password=vm_pwd)
return res_srv
def run_udhcpc_on_vm(self, vm):
command = 'sudo -i cirros-dhcpc up eth0'
result = network_checks.run_on_vm(self.env, self.os_conn, vm,
self.instance_keypair, command)
err_msg = 'Failed to start the udhcpc on vm std_err: {}'.format(
result['stderr'])
assert not result['exit_code'], err_msg
def get_lost():
result = network_checks.run_on_vm(self.env,
self.os_conn,
self.server1,
self.instance_keypair,
cmd,
vm_login='******',
vm_password='******')
return self.get_lost_percentage(result['stdout'])
def launch_iperf_server(self, vm, keypair, vm_login, vm_pwd):
"""Launch iperf server"""
server_cmd = 'iperf -u -s -p 5002 </dev/null > ~/iperf.log 2>&1 &'
res_srv = network_checks.run_on_vm(self.env,
self.os_conn,
vm,
keypair,
server_cmd,
vm_login=vm_login,
vm_password=vm_pwd)
return res_srv
def launch_iperf_client(self, client, server, keypair, vm_login,
vm_pwd, background=False):
"""Launch iperf client"""
client_cmd = 'iperf --port 5002 -u --client {0} --len 64' \
' --bandwidth 1M --time 60 -i 10' \
.format(self.os_conn.get_nova_instance_ips(server)['fixed'])
if background:
client_cmd += ' < /dev/null > ~/iperf_client.log 2>&1 &'
res = network_checks.run_on_vm(self.env, self.os_conn, client, keypair,
client_cmd, vm_login=vm_login,
vm_password=vm_pwd)
return res
def launch_iperf_client(self,
client,
server,
keypair,
vm_login,
vm_pwd,
background=False):
"""Launch iperf client"""
client_cmd = 'iperf --port 5002 -u --client {0} --len 64' \
' --bandwidth 1M --time 60 -i 10' \
.format(self.os_conn.get_nova_instance_ips(server)['fixed'])
if background:
client_cmd += ' < /dev/null > ~/iperf_client.log 2>&1 &'
res = network_checks.run_on_vm(self.env,
self.os_conn,
client,
keypair,
client_cmd,
vm_login=vm_login,
vm_password=vm_pwd)
return res
def test_broadcast_traffic_propagation_single_net(self, router):
"""Check broadcast traffic between instances placed in a single
private network and hosted on different nodes
Scenario:
1. Create private network net01, subnet 10.1.1.0/24
2. Create private network net02, subnet 10.1.2.0/24
3. Create router01_02 and connect net01 and net02 with it
4. Boot instances vm1 in net01 and compute1
5. Boot instances vm2 in net01 and compute2
6. Boot instances vm3 in net02 and compute2
7. On the compute2 start listen traffic from vm1 fixed ip on
tap interface of vm3
8. Go to the vm1's console and initiate broadcast traffic to vm3
9. Check that ARP traffic appears on listened interface
10. On the compute2 start listen traffic from vm1 fixed ip om
tap interface of vm2
11. Go to the vm1's console and initiate broadcast traffic to vm3
12. Check that ARP traffic is absent on listened interface
"""
# Create networks
compute_nodes = self.zone.hosts.keys()[:2]
net1, subnet1 = self.create_internal_network_with_subnet(suffix=1)
net2, subnet2 = self.create_internal_network_with_subnet(suffix=2)
for subnet in (subnet1, subnet2):
self.os_conn.router_interface_add(
router_id=router['router']['id'],
subnet_id=subnet['subnet']['id'])
# Create instances:
for i, (node, net) in enumerate(((compute_nodes[0], net1),
(compute_nodes[1], net1),
(compute_nodes[1], net2)), 1):
self.os_conn.create_server(
name='server%02d' % i,
availability_zone='{}:{}'.format(self.zone.zoneName, node),
key_name=self.instance_keypair.name,
nics=[{'net-id': net['network']['id']}],
security_groups=[self.security_group.id])
server1 = self.os_conn.nova.servers.find(name="server01")
server1_ip = self.os_conn.get_nova_instance_ips(server1)['fixed']
server2 = self.os_conn.nova.servers.find(name="server02")
server2_ip = self.os_conn.get_nova_instance_ips(server2)['fixed']
server3 = self.os_conn.nova.servers.find(name="server03")
server3_ip = self.os_conn.get_nova_instance_ips(server3)['fixed']
compute2 = self.env.find_node_by_fqdn(compute_nodes[1])
server2_port = self.os_conn.get_port_by_fixed_ip(server2_ip)
server2_tap = 'tap{}'.format(server2_port['id'][:11])
# Initiate broadcast traffic from server1 to server2
broadcast_log_file = gen_temp_file(prefix='broadcast', suffix='.log')
with tcpdump(
ip=compute2.data['ip'], env=self.env,
log_path=broadcast_log_file.name,
tcpdump_args=' -n src host {ip} -i {interface}'.format(
ip=server1_ip,
interface=server2_tap,)
):
cmd = 'sudo /usr/sbin/arping -I eth0 -c 4 {0}; true'.format(
server2_ip)
network_checks.run_on_vm(self.env, self.os_conn, server1,
self.instance_keypair, cmd)
check_arp_traffic(src_ip=server1_ip, dst_ip=server2_ip,
log_file=broadcast_log_file.name)
server3_port = self.os_conn.get_port_by_fixed_ip(server3_ip)
server3_tap = 'tap{}'.format(server3_port['id'][:11])
# Initiate broadcast traffic from server1 to server3
with tcpdump(
ip=compute2.data['ip'], env=self.env,
log_path=broadcast_log_file.name,
tcpdump_args=' -n src host {ip} -i {interface}'.format(
ip=server1_ip,
interface=server3_tap,)
):
cmd = 'sudo /usr/sbin/arping -I eth0 -c 4 {0}; true'.format(
server2_ip)
network_checks.run_on_vm(self.env, self.os_conn, server1,
self.instance_keypair, cmd)
check_no_arp_traffic(src_ip=server1_ip, dst_ip=server2_ip,
log_file=broadcast_log_file.name)
def test_broadcast_traffic_propagation(self, router, tcpdump_args):
"""Check broadcast traffic propagation for network segments
Scenario:
1. Create private network net01, subnet 10.1.1.0/24
2. Create private network net02, subnet 10.1.2.0/24
3. Create router01_02 and connect net01 and net02 with it
4. Boot instances vm1 in net01 and vm2 in net02
on different computes
5. Check that net02 got a new segmentation_id, different from net1
6. Go to the vm1's console and initiate broadcast traffic to vm2
7. On the compute where vm2 is hosted start listen vxlan port 4789
8. Check that no ARP traffic associated with vm1-vm2 pair
appears on compute node's console
9. Go to the vm1's console, stop arping and initiate
unicast traffic to vm2
10. Check that ICMP unicast traffic associated with vm1-vm2 pair
was captured on compute node's console
"""
# Create network and instance
compute_nodes = self.zone.hosts.keys()[:2]
for i, compute_node in enumerate(compute_nodes, 1):
network = self.os_conn.create_network(name='net%02d' % i)
subnet = self.os_conn.create_subnet(
network_id=network['network']['id'],
name='net%02d__subnet' % i,
cidr="10.1.%d.0/24" % i)
self.os_conn.router_interface_add(
router_id=router['router']['id'],
subnet_id=subnet['subnet']['id'])
self.os_conn.create_server(
name='server%02d' % i,
availability_zone='{}:{}'.format(self.zone.zoneName,
compute_node),
key_name=self.instance_keypair.name,
nics=[{'net-id': network['network']['id']}],
security_groups=[self.security_group.id])
net1, net2 = [x for x in self.os_conn.list_networks()['networks']
if x['name'] in ("net01", "net02")]
# Check that networks has different segmentation_id
assert (net1['provider:segmentation_id'] !=
net2['provider:segmentation_id'])
server1 = self.os_conn.nova.servers.find(name="server01")
server1_ip = self.os_conn.get_nova_instance_ips(server1)['fixed']
server2 = self.os_conn.nova.servers.find(name="server02")
server2_ip = self.os_conn.get_nova_instance_ips(server2)['fixed']
compute2 = self.env.find_node_by_fqdn(compute_nodes[1])
# Initiate broadcast traffic from server1 to server2
broadcast_log_file = gen_temp_file(prefix='broadcast', suffix='.log')
with tcpdump(
ip=compute2.data['ip'], env=self.env,
log_path=broadcast_log_file.name,
tcpdump_args=tcpdump_args.format(source_ip=server1_ip)
):
cmd = 'sudo /usr/sbin/arping -I eth0 -c 4 {0}; true'.format(
server2_ip)
network_checks.run_on_vm(self.env, self.os_conn, server1,
self.instance_keypair, cmd)
check_no_arp_traffic(src_ip=server1_ip, dst_ip=server2_ip,
log_file=broadcast_log_file.name)
# Initiate unicast traffic from server1 to server2
unicast_log_file = gen_temp_file(prefix='unicast', suffix='.log')
with tcpdump(
ip=compute2.data['ip'], env=self.env,
log_path=unicast_log_file.name,
tcpdump_args=tcpdump_args.format(source_ip=server1_ip)
):
cmd = 'ping -c 4 {0}; true'.format(server2_ip)
network_checks.run_on_vm(self.env, self.os_conn, server1,
self.instance_keypair, cmd)
check_icmp_traffic(src_ip=server1_ip, dst_ip=server2_ip,
log_file=unicast_log_file.name)
def test_restart_openvswitch_agent_under_bat(self):
"""Restart openvswitch-agents with broadcast traffic background
Steps:
1. Go to vm1's console and run arping
to initiate broadcast traffic:
sudo arping -I eth0 <vm2_fixed_ip>
2. Disable ovs-agents on all controllers
3. Restart service 'neutron-plugin-openvswitch-agent'
on all computes
4. Enable ovs-agents back.
5. Check that pings between vm1 and vm2 aren't interrupted
or not more than 2 packets are lost
"""
self._prepare_openstack()
# Run arping in background on server01 towards server02
srv_list = self.os_conn.nova.servers.list()
srv1 = srv_list.pop()
srv2 = srv_list.pop()
vm_ip = self.os_conn.get_nova_instance_ips(
self.os_conn.nova.servers.find(name=srv2.name))['fixed']
arping_cmd = 'sudo arping -I eth0 {}'.format(vm_ip)
cmd = ' '.join((arping_cmd, '< /dev/null > ~/arp.log 2>&1 &'))
result = network_checks.run_on_vm(self.env, self.os_conn, srv1,
self.instance_keypair, cmd)
err_msg = 'Failed to start the arping on vm result: {}'.format(result)
assert not result['exit_code'], err_msg
# Then check that all ovs agents are alive
self.os_conn.wait_agents_alive(self.ovs_agent_ids)
# Disable ovs agent on all controllers
self.disable_ovs_agents_on_controller()
# Then check that all ovs went down
self.os_conn.wait_agents_down(self.ovs_conroller_agents)
# Restart ovs agent service on all computes
self.restart_ovs_agents_on_computes()
# Enable ovs agent on all controllers
self.enable_ovs_agents_on_controllers()
# Then check that all ovs agents are alive
self.os_conn.wait_agents_alive(self.ovs_agent_ids)
# Check that arping is still executing
cmd = 'ps'
result = network_checks.run_on_vm(self.env, self.os_conn, srv1,
self.instance_keypair, cmd)
arping_is_run = False
for line in result['stdout']:
if arping_cmd in line:
arping_is_run = True
break
err_msg = 'arping was not found in stdout: {}'.format(result['stdout'])
assert arping_is_run, err_msg
# Read log of arpping execution for future possible debug
cmd = 'cat ~/arp.log'
result = network_checks.run_on_vm(self.env, self.os_conn, srv1,
self.instance_keypair, cmd)
logger.debug(result)
# Check connectivity
network_checks.check_vm_connectivity(self.env, self.os_conn)
def get_lost():
result = network_checks.run_on_vm(
self.env, self.os_conn, self.server1, self.instance_keypair,
cmd, vm_login='******', vm_password='******')
return self.get_lost_percentage(result['stdout'])
def test_restart_openvswitch_agent_under_bat(self):
"""Restart openvswitch-agents with broadcast traffic background
Steps:
1. Go to vm1's console and run arping
to initiate broadcast traffic:
sudo arping -I eth0 <vm2_fixed_ip>
2. Disable ovs-agents on all controllers
3. Restart service 'neutron-plugin-openvswitch-agent'
on all computes
4. Enable ovs-agents back.
5. Check that pings between vm1 and vm2 aren't interrupted
or not more than 2 packets are lost
"""
self._prepare_openstack()
# Run arping in background on server01 towards server02
srv_list = self.os_conn.nova.servers.list()
srv1 = srv_list.pop()
srv2 = srv_list.pop()
vm_ip = self.os_conn.get_nova_instance_ips(
self.os_conn.nova.servers.find(name=srv2.name))['fixed']
arping_cmd = 'sudo arping -I eth0 {}'.format(vm_ip)
cmd = ' '.join((arping_cmd, '< /dev/null > ~/arp.log 2>&1 &'))
result = network_checks.run_on_vm(self.env, self.os_conn, srv1,
self.instance_keypair, cmd)
err_msg = 'Failed to start the arping on vm result: {}'.format(
result)
assert not result['exit_code'], err_msg
# Then check that all ovs agents are alive
self.os_conn.wait_agents_alive(self.ovs_agent_ids)
# Disable ovs agent on all controllers
self.disable_ovs_agents_on_controller()
# Then check that all ovs went down
self.os_conn.wait_agents_down(self.ovs_conroller_agents)
# Restart ovs agent service on all computes
self.restart_ovs_agents_on_computes()
# Enable ovs agent on all controllers
self.enable_ovs_agents_on_controllers()
# Then check that all ovs agents are alive
self.os_conn.wait_agents_alive(self.ovs_agent_ids)
# Check that arping is still executing
cmd = 'ps'
result = network_checks.run_on_vm(self.env, self.os_conn, srv1,
self.instance_keypair, cmd)
arping_is_run = False
for line in result['stdout']:
if arping_cmd in line:
arping_is_run = True
break
err_msg = 'arping was not found in stdout: {}'.format(result['stdout'])
assert arping_is_run, err_msg
# Read log of arpping execution for future possible debug
cmd = 'cat ~/arp.log'
result = network_checks.run_on_vm(self.env, self.os_conn, srv1,
self.instance_keypair, cmd)
logger.debug(result)
# Check connectivity
network_checks.check_vm_connectivity(self.env, self.os_conn)
def test_broadcast_traffic_propagation_single_net(self, router):
"""Check broadcast traffic between instances placed in a single
private network and hosted on different nodes
Scenario:
1. Create private network net01, subnet 10.1.1.0/24
2. Create private network net02, subnet 10.1.2.0/24
3. Create router01_02 and connect net01 and net02 with it
4. Boot instances vm1 in net01 and compute1
5. Boot instances vm2 in net01 and compute2
6. Boot instances vm3 in net02 and compute2
7. On the compute2 start listen traffic from vm1 fixed ip on
tap interface of vm3
8. Go to the vm1's console and initiate broadcast traffic to vm3
9. Check that ARP traffic appears on listened interface
10. On the compute2 start listen traffic from vm1 fixed ip om
tap interface of vm2
11. Go to the vm1's console and initiate broadcast traffic to vm3
12. Check that ARP traffic is absent on listened interface
"""
# Create networks
compute_nodes = self.zone.hosts.keys()[:2]
net1, subnet1 = self.create_internal_network_with_subnet(suffix=1)
net2, subnet2 = self.create_internal_network_with_subnet(suffix=2)
for subnet in (subnet1, subnet2):
self.os_conn.router_interface_add(router_id=router['router']['id'],
subnet_id=subnet['subnet']['id'])
# Create instances:
for i, (node, net) in enumerate(
((compute_nodes[0], net1), (compute_nodes[1], net1),
(compute_nodes[1], net2)), 1):
self.os_conn.create_server(
name='server%02d' % i,
availability_zone='{}:{}'.format(self.zone.zoneName, node),
key_name=self.instance_keypair.name,
nics=[{
'net-id': net['network']['id']
}],
security_groups=[self.security_group.id])
server1 = self.os_conn.nova.servers.find(name="server01")
server1_ip = self.os_conn.get_nova_instance_ips(server1)['fixed']
server2 = self.os_conn.nova.servers.find(name="server02")
server2_ip = self.os_conn.get_nova_instance_ips(server2)['fixed']
server3 = self.os_conn.nova.servers.find(name="server03")
server3_ip = self.os_conn.get_nova_instance_ips(server3)['fixed']
compute2 = self.env.find_node_by_fqdn(compute_nodes[1])
server2_port = self.os_conn.get_port_by_fixed_ip(server2_ip)
server2_tap = 'tap{}'.format(server2_port['id'][:11])
# Initiate broadcast traffic from server1 to server2
broadcast_log_file = gen_temp_file(prefix='broadcast', suffix='.log')
with tcpdump(ip=compute2.data['ip'],
env=self.env,
log_path=broadcast_log_file.name,
tcpdump_args=' -n src host {ip} -i {interface}'.format(
ip=server1_ip,
interface=server2_tap,
)):
cmd = 'sudo arping -I eth0 -c 4 {0}; true'.format(server2_ip)
network_checks.run_on_vm(self.env, self.os_conn, server1,
self.instance_keypair, cmd)
check_arp_traffic(src_ip=server1_ip,
dst_ip=server2_ip,
log_file=broadcast_log_file.name)
server3_port = self.os_conn.get_port_by_fixed_ip(server3_ip)
server3_tap = 'tap{}'.format(server3_port['id'][:11])
# Initiate broadcast traffic from server1 to server3
with tcpdump(ip=compute2.data['ip'],
env=self.env,
log_path=broadcast_log_file.name,
tcpdump_args=' -n src host {ip} -i {interface}'.format(
ip=server1_ip,
interface=server3_tap,
)):
cmd = 'sudo arping -I eth0 -c 4 {0}; true'.format(server2_ip)
network_checks.run_on_vm(self.env, self.os_conn, server1,
self.instance_keypair, cmd)
check_no_arp_traffic(src_ip=server1_ip,
dst_ip=server2_ip,
log_file=broadcast_log_file.name)
def test_broadcast_traffic_propagation(self, router, tcpdump_args):
"""Check broadcast traffic propagation for network segments
Scenario:
1. Create private network net01, subnet 10.1.1.0/24
2. Create private network net02, subnet 10.1.2.0/24
3. Create router01_02 and connect net01 and net02 with it
4. Boot instances vm1 in net01 and vm2 in net02
on different computes
5. Check that net02 got a new segmentation_id, different from net1
6. Go to the vm1's console and initiate broadcast traffic to vm2
7. On the compute where vm2 is hosted start listen vxlan port 4789
8. Check that no ARP traffic associated with vm1-vm2 pair
appears on compute node's console
9. Go to the vm1's console, stop arping and initiate
unicast traffic to vm2
10. Check that ICMP unicast traffic associated with vm1-vm2 pair
was captured on compute node's console
"""
# Create network and instance
compute_nodes = self.zone.hosts.keys()[:2]
for i, compute_node in enumerate(compute_nodes, 1):
network = self.os_conn.create_network(name='net%02d' % i)
subnet = self.os_conn.create_subnet(
network_id=network['network']['id'],
name='net%02d__subnet' % i,
cidr="10.1.%d.0/24" % i)
self.os_conn.router_interface_add(router_id=router['router']['id'],
subnet_id=subnet['subnet']['id'])
self.os_conn.create_server(
name='server%02d' % i,
availability_zone='{}:{}'.format(self.zone.zoneName,
compute_node),
key_name=self.instance_keypair.name,
nics=[{
'net-id': network['network']['id']
}],
security_groups=[self.security_group.id])
net1, net2 = [
x for x in self.os_conn.list_networks()['networks']
if x['name'] in ("net01", "net02")
]
# Check that networks has different segmentation_id
assert (net1['provider:segmentation_id'] !=
net2['provider:segmentation_id'])
server1 = self.os_conn.nova.servers.find(name="server01")
server1_ip = self.os_conn.get_nova_instance_ips(server1)['fixed']
server2 = self.os_conn.nova.servers.find(name="server02")
server2_ip = self.os_conn.get_nova_instance_ips(server2)['fixed']
compute2 = self.env.find_node_by_fqdn(compute_nodes[1])
# Initiate broadcast traffic from server1 to server2
broadcast_log_file = gen_temp_file(prefix='broadcast', suffix='.log')
with tcpdump(ip=compute2.data['ip'],
env=self.env,
log_path=broadcast_log_file.name,
tcpdump_args=tcpdump_args.format(source_ip=server1_ip)):
cmd = 'sudo arping -I eth0 -c 4 {0}; true'.format(server2_ip)
network_checks.run_on_vm(self.env, self.os_conn, server1,
self.instance_keypair, cmd)
check_no_arp_traffic(src_ip=server1_ip,
dst_ip=server2_ip,
log_file=broadcast_log_file.name)
# Initiate unicast traffic from server1 to server2
unicast_log_file = gen_temp_file(prefix='unicast', suffix='.log')
with tcpdump(ip=compute2.data['ip'],
env=self.env,
log_path=unicast_log_file.name,
tcpdump_args=tcpdump_args.format(source_ip=server1_ip)):
cmd = 'ping -c 4 {0}; true'.format(server2_ip)
network_checks.run_on_vm(self.env, self.os_conn, server1,
self.instance_keypair, cmd)
check_icmp_traffic(src_ip=server1_ip,
dst_ip=server2_ip,
log_file=unicast_log_file.name)