def wait_until_none(predicate, timeout=5, sleep=1, exception=None):
def internal_predicate():
ret = predicate()
if ret:
return False
return True
wait_until_true(internal_predicate, timeout, sleep, exception)
def test_has_updates(self):
utils.wait_until_true(lambda: self.monitor.has_updates)
# clear the event list
self.monitor.get_events()
self.useFixture(net_helpers.OVSPortFixture())
# has_updates after port addition should become True
utils.wait_until_true(lambda: self.monitor.has_updates is True)
def _spawn_keepalived(self, keepalived_manager):
keepalived_manager.spawn()
process = keepalived_manager.get_process()
utils.wait_until_true(
lambda: process.active, timeout=5, sleep=0.01, exception=RuntimeError(_("Keepalived didn't spawn"))
)
return process
def _read_stream(stream, timeout):
if timeout:
poller = select.poll()
poller.register(stream.fileno())
poll_predicate = functools.partial(poller.poll, 1)
utils.wait_until_true(poll_predicate, timeout, 0.1, RuntimeError("No output in %.2f seconds" % timeout))
return stream.readline()
def setUp(self):
super(NeutronServerFixture, self).setUp()
self.neutron_cfg_fixture = config_fixtures.NeutronConfigFixture(
self.temp_dir, cfg.CONF.database.connection,
self.rabbitmq_environment)
self.plugin_cfg_fixture = config_fixtures.ML2ConfigFixture(
self.temp_dir)
self.useFixture(self.neutron_cfg_fixture)
self.useFixture(self.plugin_cfg_fixture)
self.neutron_config = self.neutron_cfg_fixture.config
self.plugin_config = self.plugin_cfg_fixture.config
config_filenames = [self.neutron_cfg_fixture.filename,
self.plugin_cfg_fixture.filename]
self.process_fixture = self.useFixture(ProcessFixture(
test_name=self.test_name,
process_name=self.NEUTRON_SERVER,
exec_name=self.NEUTRON_SERVER,
config_filenames=config_filenames))
utils.wait_until_true(self.server_is_live)
def setup_ha_routers(self, router_public_ip, private_net_cidr):
"""Setup HA master router on agent1 and backup router on agent2"""
router_info = self._generate_info(router_public_ip,
private_net_cidr, enable_ha=True)
get_ns_name = mock.patch.object(
namespaces.RouterNamespace, '_get_ns_name').start()
get_ns_name.return_value = "qrouter-{0}-{1}".format(
router_info['id'], self.vpn_agent.host)
router1 = self.manage_router(self.vpn_agent, router_info)
router_info_2 = copy.deepcopy(router_info)
router_info_2[l3_constants.HA_INTERFACE_KEY] = (
l3_test_common.get_ha_interface(ip='169.254.192.2',
mac='22:22:22:22:22:22'))
get_ns_name.return_value = "qrouter-{0}-{1}".format(
router_info['id'], self.failover_agent.host)
router2 = self.manage_router(self.failover_agent, router_info_2)
linux_utils.wait_until_true(lambda: router1.ha_state == 'master')
linux_utils.wait_until_true(lambda: router2.ha_state == 'backup')
port_namespace, port_ip = self.port_setup(router1)
vpn_service = self.prepare_vpn_service_info(
router1.router_id, router_public_ip, private_net_cidr)
return {"router1": router1, "router2": router2,
"port_namespace": port_namespace, "port_ip": port_ip,
"vpn_service": vpn_service}
def test_dhcp_app_dos_block(self):
def internal_predicate():
ovs = test_utils.OvsFlowsParser()
return (self._check_dhcp_block_rule(ovs.dump()))
dhcp_packet = self._create_dhcp_discover()
send_dhcp_offer = app_testing_objects.SendAction(
self.subnet1.subnet_id,
self.port1.port_id,
str(dhcp_packet)
)
port_policies = self._create_port_policies(disable_rule=False)
policy = self.store(
app_testing_objects.Policy(
initial_actions=[send_dhcp_offer,
send_dhcp_offer,
send_dhcp_offer,
send_dhcp_offer],
port_policies=port_policies,
unknown_port_action=app_testing_objects.IgnoreAction()
)
)
policy.start(self.topology)
wait_until_true(internal_predicate, 30, 1, None)
def _test_restart_service_on_sighup(self, service, workers=1):
"""Test that a service correctly (re)starts on receiving SIGHUP.
1. Start a service with a given number of workers.
2. Send SIGHUP to the service.
3. Wait for workers (if any) to (re)start.
"""
self._start_server(callback=service, workers=workers)
os.kill(self.service_pid, signal.SIGHUP)
expected_msg = FAKE_START_MSG * workers * 2
# Wait for temp file to be created and its size reaching the expected
# value
expected_size = len(expected_msg)
condition = lambda: (os.path.isfile(self.temp_file)
and os.stat(self.temp_file).st_size ==
expected_size)
utils.wait_until_true(
condition, timeout=5, sleep=0.1,
exception=RuntimeError(
"Timed out waiting for file %(filename)s to be created and "
"its size become equal to %(size)s." %
{'filename': self.temp_file,
'size': expected_size}))
# Verify that start has been called twice for each worker (one for
# initial start, and the second one on SIGHUP after children were
# terminated).
with open(self.temp_file, 'r') as f:
res = f.readline()
self.assertEqual(expected_msg, res)
def wait_until_fip_active(self, timeout=5, sleep=1, exception=None):
def internal_predicate():
fip = self.get_floatingip()
if fip and fip.get_status() == 'ACTIVE':
return True
return False
wait_until_true(internal_predicate, timeout, sleep, exception)
def create_site(self, public_net, private_nets, l3ha=False):
"""Build router(s), namespaces, and ports for a site.
For HA, we'll create a backup router and wait for both routers
to be ready, so that we can test pings after failover.
"""
if l3ha:
site = SiteInfoWithHaRouter(public_net, private_nets,
self.vpn_agent.host,
self.failover_agent.host)
else:
site = SiteInfo(public_net, private_nets)
site.router = self.create_router(self.vpn_agent, site.info)
if l3ha:
backup_info = site.generate_backup_router_info()
site.backup_router = self.create_router(self.failover_agent,
backup_info)
linux_utils.wait_until_true(
lambda: site.router.ha_state == 'master')
linux_utils.wait_until_true(
lambda: site.backup_router.ha_state == 'backup')
self.create_ports_for(site)
return site
def _wait_for_death(self):
is_dead = lambda: self.proc.poll() is not None
utils.wait_until_true(
is_dead,
timeout=self.TIMEOUT,
exception=RuntimeError("Ping command hasn't ended after %d seconds." % self.TIMEOUT),
)
def _assert_router_does_not_exist(self, router):
# If the namespace assertion succeeds
# then the devices and iptable rules have also been deleted,
# so there's no need to check that explicitly.
self.assertFalse(self._namespace_exists(router.ns_name))
utils.wait_until_true(
lambda: not self._metadata_proxy_exists(self.agent.conf, router))
def test_create_update_subnet_with_dhcp(self):
ovs = utils.OvsFlowsParser()
flows_before_change = ovs.dump(self.integration_bridge)
network = self.store(objects.NetworkTestObj(self.neutron, self.nb_api))
network_id = network.create()
subnet = {'network_id': network_id,
'cidr': '10.10.254.0/24',
'gateway_ip': '10.10.254.1',
'ip_version': 4,
'name': 'subnet-test',
'enable_dhcp': True}
subnet = self.neutron.create_subnet({'subnet': subnet})
subnet_id = subnet['subnet']['id']
dhcp_ip = utils.wait_until_is_and_return(
lambda: self.get_dhcp_ip(network_id, subnet_id),
exception=Exception('DHCP IP was not generated')
)
self.assertFalse(self.check_dhcp_rule(flows_before_change, dhcp_ip))
wait_until_true(
lambda: self.check_dhcp_rule(ovs.dump(self.integration_bridge),
dhcp_ip),
exception=Exception('DHCP ip was not found in OpenFlow rules'),
timeout=5
)
# change dhcp
updated_subnet = {'enable_dhcp': False}
self.neutron.update_subnet(subnet_id, {'subnet': updated_subnet})
time.sleep(utils.DEFAULT_CMD_TIMEOUT)
flows_after_update = ovs.dump(self.integration_bridge)
self.assertFalse(self.check_dhcp_rule(flows_after_update, dhcp_ip))
network.close()
def _assert_dvr_external_device(self, router):
external_port = router.get_ex_gw_port()
snat_ns_name = dvr_snat_ns.SnatNamespace.get_snat_ns_name(
router.router_id)
# if the agent is in dvr_snat mode, then we have to check
# that the correct ports and ip addresses exist in the
# snat_ns_name namespace
if self.agent.conf.agent_mode == 'dvr_snat':
device_exists = functools.partial(
self.device_exists_with_ips_and_mac,
external_port,
router.get_external_device_name,
snat_ns_name)
utils.wait_until_true(device_exists)
# if the agent is in dvr mode then the snat_ns_name namespace
# should not be present at all:
elif self.agent.conf.agent_mode == 'dvr':
self.assertFalse(
self._namespace_exists(snat_ns_name),
"namespace %s was found but agent is in dvr mode not dvr_snat"
% (str(snat_ns_name))
)
# if the agent is anything else the test is misconfigured
# we force a test failure with message
else:
self.assertTrue(False, " agent not configured for dvr or dvr_snat")
def test_ha_router_conf_on_restarted_agent(self):
router_info = self.generate_router_info(enable_ha=True)
router1 = self._create_router(self.agent, router_info)
self._add_fip(router1, '192.168.111.12')
restarted_agent = l3_test_agent.TestL3NATAgent(self.agent.host,
self.agent.conf)
self._create_router(restarted_agent, router1.router)
utils.wait_until_true(lambda: self._floating_ips_configured(router1))
def test_has_updates(self):
utils.wait_until_true(lambda: self.monitor.data_received is True)
self.assertTrue(self.monitor.has_updates, "Initial call should always be true")
# clear the event list
self.monitor.get_events()
self.useFixture(net_helpers.OVSPortFixture())
# has_updates after port addition should become True
utils.wait_until_true(lambda: self.monitor.has_updates is True)
def _ensure_lla_task(self, gw_ifname, ns_name, lla_with_mask):
# It would be insane for taking so long unless DAD test failed
# In that case, the subnet would never be assigned a prefix.
linux_utils.wait_until_true(functools.partial(self._lla_available,
gw_ifname,
ns_name,
lla_with_mask),
timeout=l3_constants.LLA_TASK_TIMEOUT,
sleep=2)
def _wait_for_ipsec_startup(self, router, driver, conf, should_run=True):
"""Wait for new IPSec process on failover agent to start up."""
# check for both strongswan and openswan processes
path = driver.processes[router.router_id].config_dir
pid_files = ['%s/var/run/charon.pid' % path,
'%s/var/run/pluto.pid' % path]
linux_utils.wait_until_true(
lambda: should_run == self._ipsec_process_exists(
conf, router, pid_files))
def wait_until_bandwidth_limit_rule_applied(bridge, port_vif, rule):
def _bandwidth_limit_rule_applied():
bw_rule = bridge.get_egress_bw_limit_for_port(port_vif)
expected = None, None
if rule:
expected = rule.max_kbps, rule.max_burst_kbps
return bw_rule == expected
agent_utils.wait_until_true(_bandwidth_limit_rule_applied)
def start_subscribing(self):
self._topology = self.store(
test_objects.Topology(self.neutron, self.nb_api))
subnet = self._topology.create_subnet(cidr="192.168.200.0/24")
port = subnet.create_port()
wait_until_true(
lambda: port.name is not None, timeout=30,
exception=Exception('Port was not created')
)
def wait_until_is_and_return(predicate, timeout=5, sleep=1, exception=None):
container = {}
def internal_predicate():
container['value'] = predicate()
return container['value']
wait_until_true(internal_predicate, timeout, sleep, exception)
return container.get('value')
def test_keepalived_respawns(self):
self.manager.spawn()
process = self.manager.get_process()
self.assertTrue(process.active)
process.disable(sig="15")
utils.wait_until_true(
lambda: process.active, timeout=5, sleep=0.01, exception=RuntimeError(_("Keepalived didn't respawn"))
)
def test_has_updates(self):
utils.wait_until_true(lambda: self.monitor.data_received is True)
self.assertTrue(self.monitor.has_updates,
'Initial call should always be true')
self.assertFalse(self.monitor.has_updates,
'has_updates without port addition should be False')
self.useFixture(net_helpers.OVSPortFixture())
# has_updates after port addition should become True
while not self.monitor.has_updates:
eventlet.sleep(0.01)
def test_resync_devices_set_up_after_exception(self):
agent = self.create_agent()
self.start_agent(agent)
network = self._create_test_network_dict()
ports = self._create_ports(network, agent, True)
self._plug_ports(network, ports, agent)
ports_ids = [p['id'] for p in ports]
agent_utils.wait_until_true(
lambda: self._expected_plugin_rpc_call(
agent.plugin_rpc.update_device_list, ports_ids))
def test_ha_router_conf_on_restarted_agent(self):
router_info = self.generate_router_info(enable_ha=True)
router1 = self.manage_router(self.agent, router_info)
self._add_fip(router1, "192.168.111.12")
restarted_agent = neutron_l3_agent.L3NATAgentWithStateReport(self.agent.host, self.agent.conf)
self.manage_router(restarted_agent, router1.router)
utils.wait_until_true(lambda: self.floating_ips_configured(router1))
self.assertIn(
router1._get_primary_vip(), self._get_addresses_on_device(router1.ns_name, router1.get_ha_device_name())
)
def assert_patch_ports(self, agent):
def get_peer(port):
return agent.int_br.db_get_val(
'Interface', port, 'options', check_error=True)
agent_utils.wait_until_true(
lambda: get_peer(self.patch_int) == {'peer': self.patch_tun})
agent_utils.wait_until_true(
lambda: get_peer(self.patch_tun) == {'peer': self.patch_int})
def verify_ipv6_address_assignment(self, gw_dev):
process = self.manager.get_process()
agent_utils.wait_until_true(lambda: process.active)
def _gw_vip_assigned():
iface_ip = gw_dev.addr.list(ip_version=6, scope='global')
if iface_ip:
return self.gw_vip == iface_ip[0]['cidr']
agent_utils.wait_until_true(_gw_vip_assigned)
def _wait_for_bw_rule_applied_linuxbridge_agent(self, vm, limit, burst):
port_name = linuxbridge_agent.LinuxBridgeManager.get_tap_device_name(
vm.neutron_port['id'])
tc = tc_lib.TcCommand(
port_name,
linuxbridge_agent_config.DEFAULT_KERNEL_HZ_VALUE,
namespace=vm.host.host_namespace
)
utils.wait_until_true(
lambda: tc.get_filters_bw_limits() == (limit, burst))
def test_metadata_proxy_respawned(self):
pm, network = self._spawn_network_metadata_proxy()
old_pid = pm.pid
utils.execute(['kill', '-9', old_pid], run_as_root=True)
utils.wait_until_true(
lambda: pm.active and pm.pid != old_pid,
timeout=5,
sleep=0.1,
exception=RuntimeError("Metadata proxy didn't respawn"))
def test_stale_metadata_proxy_killed(self):
pm, network = self._spawn_network_metadata_proxy()
self.conf.set_override('enable_isolated_metadata', False)
self.configure_dhcp_for_network(network=network)
utils.wait_until_true(
lambda: not pm.active,
timeout=5,
sleep=0.1,
exception=RuntimeError("Stale metadata proxy didn't get killed"))
def _assert_dvr_external_device(self, router):
external_port = router.get_ex_gw_port()
snat_ns_name = dvr_snat_ns.SnatNamespace.get_snat_ns_name(
router.router_id)
# if the agent is in dvr_snat mode, then we have to check
# that the correct ports and ip addresses exist in the
# snat_ns_name namespace
if self.agent.conf.agent_mode == 'dvr_snat':
device_exists = functools.partial(
self.device_exists_with_ips_and_mac, external_port,
router.get_external_device_name, snat_ns_name)
utils.wait_until_true(device_exists)
# if the agent is in dvr mode then the snat_ns_name namespace
# should not be present at all:
elif self.agent.conf.agent_mode == 'dvr':
self.assertFalse(
self._namespace_exists(snat_ns_name),
"namespace %s was found but agent is in dvr mode not dvr_snat"
% (str(snat_ns_name)))
# if the agent is anything else the test is misconfigured
# we force a test failure with message
else:
self.assertTrue(False, " agent not configured for dvr or dvr_snat")
def setUp(self):
super(NeutronServerFixture, self).setUp()
self.neutron_cfg_fixture = config_fixtures.NeutronConfigFixture(
self.temp_dir, cfg.CONF.database.connection)
self.plugin_cfg_fixture = config_fixtures.ML2ConfigFixture(
self.temp_dir)
self.useFixture(self.neutron_cfg_fixture)
self.useFixture(self.plugin_cfg_fixture)
self.neutron_config = self.neutron_cfg_fixture.config
self.plugin_config = self.plugin_cfg_fixture.config
config_filenames = [
self.neutron_cfg_fixture.filename, self.plugin_cfg_fixture.filename
]
self.process_fixture = self.useFixture(
ProcessFixture(name=self.NEUTRON_SERVER,
exec_name=self.NEUTRON_SERVER,
config_filenames=config_filenames))
utils.wait_until_true(self.server_is_live)
def test_keepalived_state_change_bulk_rpc(self):
router_info = self.generate_router_info(enable_ha=True)
router1 = self.manage_router(self.agent, router_info)
self.fail_ha_router(router1)
router_info = self.generate_router_info(enable_ha=True)
router2 = self.manage_router(self.agent, router_info)
utils.wait_until_true(lambda: router1.ha_state == 'backup')
utils.wait_until_true(lambda: router2.ha_state == 'master')
utils.wait_until_true(
lambda: self._expected_rpc_report(
{router1.router_id: 'standby', router2.router_id: 'active'}))
def _test_dvr_ha_router_failover(self, enable_gw):
self._setup_dvr_ha_agents()
self._setup_dvr_ha_bridges()
router1 = self._create_dvr_ha_router(self.agent, enable_gw=enable_gw)
router2 = self._create_dvr_ha_router(self.failover_agent, enable_gw)
utils.wait_until_true(lambda: router1.ha_state == 'master')
utils.wait_until_true(lambda: router2.ha_state == 'backup')
self._assert_ip_addresses_in_dvr_ha_snat_namespace(router1)
self._assert_no_ip_addresses_in_dvr_ha_snat_namespace(router2)
self.fail_ha_router(router1)
utils.wait_until_true(lambda: router2.ha_state == 'master')
utils.wait_until_true(lambda: router1.ha_state == 'backup')
self._assert_ip_addresses_in_dvr_ha_snat_namespace(router2)
self._assert_no_ip_addresses_in_dvr_ha_snat_namespace(router1)
def _router_lifecycle(self, enable_ha, ip_version=4):
router_info = self.generate_router_info(enable_ha, ip_version)
router = self.manage_router(self.agent, router_info)
if enable_ha:
port = router.get_ex_gw_port()
interface_name = self.agent.get_external_device_name(port['id'])
self._assert_no_ip_addresses_on_interface(router.ns_name,
interface_name)
utils.wait_until_true(lambda: router.ha_state == 'master')
# Keepalived notifies of a state transition when it starts,
# not when it ends. Thus, we have to wait until keepalived finishes
# configuring everything. We verify this by waiting until the last
# device has an IP address.
device = router.router[l3_constants.INTERFACE_KEY][-1]
device_exists = functools.partial(self.device_exists_with_ip_mac,
device,
router.get_internal_device_name,
router.ns_name)
utils.wait_until_true(device_exists)
self.assertTrue(self._namespace_exists(router.ns_name))
utils.wait_until_true(
lambda: self._metadata_proxy_exists(self.agent.conf, router))
self._assert_internal_devices(router)
self._assert_external_device(router)
if ip_version == 4:
# Note(SridharG): enable the assert_gateway for IPv6 once
# keepalived on Ubuntu14.04 (i.e., check-neutron-dsvm-functional
# platform) is updated to 1.2.10 (or above).
# For more details: https://review.openstack.org/#/c/151284/
self._assert_gateway(router)
self.assertTrue(self.floating_ips_configured(router))
self._assert_snat_chains(router)
self._assert_floating_ip_chains(router)
self._assert_extra_routes(router)
self._assert_metadata_chains(router)
if enable_ha:
self._assert_ha_device(router)
self.assertTrue(router.keepalived_manager.get_process().active)
self._delete_router(self.agent, router.router_id)
self._assert_interfaces_deleted_from_ovs()
self._assert_router_does_not_exist(router)
if enable_ha:
self.assertFalse(router.keepalived_manager.get_process().active)
def test_keepalived_state_change_notification(self):
enqueue_mock = mock.patch.object(self.agent,
'enqueue_state_change').start()
router_info = self.generate_router_info(enable_ha=True)
router = self.manage_router(self.agent, router_info)
utils.wait_until_true(lambda: router.ha_state == 'master')
self.fail_ha_router(router)
utils.wait_until_true(lambda: router.ha_state == 'backup')
utils.wait_until_true(lambda: enqueue_mock.call_count == 3)
calls = [args[0] for args in enqueue_mock.call_args_list]
self.assertEqual((router.router_id, 'backup'), calls[0])
self.assertEqual((router.router_id, 'master'), calls[1])
self.assertEqual((router.router_id, 'backup'), calls[2])
def test_ha_router_ipv6_radvd_status(self):
router_info = self.generate_router_info(ip_version=6, enable_ha=True)
router1 = self.manage_router(self.agent, router_info)
utils.wait_until_true(lambda: router1.ha_state == 'master')
utils.wait_until_true(lambda: router1.radvd.enabled)
def _check_lla_status(router, expected):
internal_devices = router.router[l3_constants.INTERFACE_KEY]
for device in internal_devices:
lladdr = ip_lib.get_ipv6_lladdr(device['mac_address'])
exists = ip_lib.device_exists_with_ips_and_mac(
router.get_internal_device_name(device['id']), [lladdr],
device['mac_address'], router.ns_name)
self.assertEqual(expected, exists)
_check_lla_status(router1, True)
device_name = router1.get_ha_device_name()
ha_device = ip_lib.IPDevice(device_name, namespace=router1.ns_name)
ha_device.link.set_down()
utils.wait_until_true(lambda: router1.ha_state == 'backup')
utils.wait_until_true(lambda: not router1.radvd.enabled, timeout=10)
_check_lla_status(router1, False)
def test_get_events(self):
utils.wait_until_true(lambda: self.monitor.has_updates)
devices = self.monitor.get_events()
self.assertTrue(devices.get('added'),
'Initial call should always be true')
br = self.useFixture(net_helpers.OVSBridgeFixture())
p1 = self.useFixture(net_helpers.OVSPortFixture(br.bridge))
p2 = self.useFixture(net_helpers.OVSPortFixture(br.bridge))
added_devices = [p1.port.name, p2.port.name]
utils.wait_until_true(
lambda: self._expected_devices_events(added_devices, 'added'))
br.bridge.delete_port(p1.port.name)
br.bridge.delete_port(p2.port.name)
removed_devices = [p1.port.name, p2.port.name]
utils.wait_until_true(
lambda: self._expected_devices_events(removed_devices, 'removed'))
# restart
self.monitor.stop(block=True)
self.monitor.start(block=True, timeout=60)
try:
utils.wait_until_true(
lambda: self.monitor.get_events().get('added'))
except eventlet.timeout.Timeout:
raise AssertionError('Initial call should always be true')
def test_keepalived_state_change_notification(self):
enqueue_mock = mock.patch.object(self.agent,
'enqueue_state_change').start()
router_info = self.generate_router_info(enable_ha=True)
router = self.manage_router(self.agent, router_info)
utils.wait_until_true(lambda: router.ha_state == 'master')
device_name = router.get_ha_device_name(
router.router[l3_constants.HA_INTERFACE_KEY]['id'])
ha_device = ip_lib.IPDevice(device_name, router.ns_name)
ha_device.link.set_down()
utils.wait_until_true(lambda: router.ha_state == 'backup')
utils.wait_until_true(lambda: enqueue_mock.call_count == 3)
calls = [args[0] for args in enqueue_mock.call_args_list]
self.assertEqual((router.router_id, 'backup'), calls[0])
self.assertEqual((router.router_id, 'master'), calls[1])
self.assertEqual((router.router_id, 'backup'), calls[2])
def _dvr_router_lifecycle(self,
enable_ha=False,
enable_snat=False,
custom_mtu=2000,
use_port_mtu=False,
ip_version=4,
dual_stack=False):
'''Test dvr router lifecycle
:param enable_ha: sets the ha value for the router.
:param enable_snat: the value of enable_snat is used
to set the agent_mode.
'''
# The value of agent_mode can be dvr, dvr_snat, or legacy.
# Since by definition this is a dvr (distributed = true)
# only dvr and dvr_snat are applicable
self.agent.conf.agent_mode = 'dvr_snat' if enable_snat else 'dvr'
# We get the router info particular to a dvr router
router_info = self.generate_dvr_router_info(enable_ha,
enable_snat,
extra_routes=True)
if use_port_mtu:
for key in ('_interfaces', '_snat_router_interfaces',
'_floatingip_agent_interfaces'):
for port in router_info[key]:
port['mtu'] = custom_mtu
router_info['gw_port']['mtu'] = custom_mtu
router_info['_ha_interface']['mtu'] = custom_mtu
else:
self.agent.conf.network_device_mtu = custom_mtu
# We need to mock the get_agent_gateway_port return value
# because the whole L3PluginApi is mocked and we need the port
# gateway_port information before the l3_agent will create it.
# The port returned needs to have the same information as
# router_info['gw_port']
self.mock_plugin_api.get_agent_gateway_port.return_value = router_info[
'gw_port']
# We also need to mock the get_external_network_id method to
# get the correct fip namespace.
self.mock_plugin_api.get_external_network_id.return_value = (
router_info['_floatingips'][0]['floating_network_id'])
# With all that set we can now ask the l3_agent to
# manage the router (create it, create namespaces,
# attach interfaces, etc...)
router = self.manage_router(self.agent, router_info)
if enable_ha:
port = router.get_ex_gw_port()
interface_name = router.get_external_device_name(port['id'])
self._assert_no_ip_addresses_on_interface(router.ha_namespace,
interface_name)
utils.wait_until_true(lambda: router.ha_state == 'master')
# Keepalived notifies of a state transition when it starts,
# not when it ends. Thus, we have to wait until keepalived finishes
# configuring everything. We verify this by waiting until the last
# device has an IP address.
device = router.router[l3_constants.INTERFACE_KEY][-1]
device_exists = functools.partial(
self.device_exists_with_ips_and_mac, device,
router.get_internal_device_name, router.ns_name)
utils.wait_until_true(device_exists)
name = router.get_internal_device_name(device['id'])
self.assertEqual(custom_mtu,
ip_lib.IPDevice(name, router.ns_name).link.mtu)
ext_gateway_port = router_info['gw_port']
self.assertTrue(self._namespace_exists(router.ns_name))
utils.wait_until_true(
lambda: self._metadata_proxy_exists(self.agent.conf, router))
self._assert_internal_devices(router)
self._assert_dvr_external_device(router)
self._assert_dvr_gateway(router)
self._assert_dvr_floating_ips(router)
self._assert_snat_chains(router)
self._assert_floating_ip_chains(router)
self._assert_metadata_chains(router)
self._assert_rfp_fpr_mtu(router, custom_mtu)
if enable_snat:
ip_versions = [4, 6] if (ip_version == 6 or dual_stack) else [4]
snat_ns_name = dvr_snat_ns.SnatNamespace.get_snat_ns_name(
router.router_id)
self._assert_onlink_subnet_routes(router, ip_versions,
snat_ns_name)
self._assert_extra_routes(router, namespace=snat_ns_name)
# During normal operation, a router-gateway-clear followed by
# a router delete results in two notifications to the agent. This
# code flow simulates the exceptional case where the notification of
# the clearing of the gateway hast been missed, so we are checking
# that the L3 agent is robust enough to handle that case and delete
# the router correctly.
self._delete_router(self.agent, router.router_id)
self._assert_fip_namespace_deleted(ext_gateway_port)
self._assert_router_does_not_exist(router)
self._assert_snat_namespace_does_not_exist(router)
def test_wait_until_true_predicate_fails(self):
with testtools.ExpectedException(eventlet.timeout.Timeout):
utils.wait_until_true(lambda: False, 2)
def test_wait_until_true_predicate_succeeds(self):
utils.wait_until_true(lambda: True)
def _assert_namespace_exists(self, ns_name):
ip = ip_lib.IPWrapper(ns_name)
utils.wait_until_true(lambda: ip.netns.exists(ns_name))
def block_until_port_status_active(self, port_id):
def is_port_status_active():
port = self.client.show_port(port_id)
return port['port']['status'] == 'ACTIVE'
utils.wait_until_true(lambda: is_port_status_active(), sleep=1)
def _router_lifecycle(self, enable_ha, ip_version=4,
dual_stack=False, v6_ext_gw_with_sub=True):
router_info = self.generate_router_info(enable_ha, ip_version,
dual_stack=dual_stack,
v6_ext_gw_with_sub=(
v6_ext_gw_with_sub))
router = self.manage_router(self.agent, router_info)
# Add multiple-IPv6-prefix internal router port
slaac = n_const.IPV6_SLAAC
slaac_mode = {'ra_mode': slaac, 'address_mode': slaac}
subnet_modes = [slaac_mode] * 2
self._add_internal_interface_by_subnet(router.router,
count=2,
ip_version=6,
ipv6_subnet_modes=subnet_modes)
router.process(self.agent)
if enable_ha:
port = router.get_ex_gw_port()
interface_name = router.get_external_device_name(port['id'])
self._assert_no_ip_addresses_on_interface(router.ns_name,
interface_name)
utils.wait_until_true(lambda: router.ha_state == 'master')
# Keepalived notifies of a state transition when it starts,
# not when it ends. Thus, we have to wait until keepalived finishes
# configuring everything. We verify this by waiting until the last
# device has an IP address.
device = router.router[l3_constants.INTERFACE_KEY][-1]
device_exists = functools.partial(
self.device_exists_with_ips_and_mac,
device,
router.get_internal_device_name,
router.ns_name)
utils.wait_until_true(device_exists)
self.assertTrue(self._namespace_exists(router.ns_name))
utils.wait_until_true(
lambda: self._metadata_proxy_exists(self.agent.conf, router))
self._assert_internal_devices(router)
self._assert_external_device(router)
if not (enable_ha and (ip_version == 6 or dual_stack)):
# Note(SridharG): enable the assert_gateway for IPv6 once
# keepalived on Ubuntu14.04 (i.e., check-neutron-dsvm-functional
# platform) is updated to 1.2.10 (or above).
# For more details: https://review.openstack.org/#/c/151284/
self._assert_gateway(router, v6_ext_gw_with_sub)
self.assertTrue(self.floating_ips_configured(router))
self._assert_snat_chains(router)
self._assert_floating_ip_chains(router)
self._assert_iptables_rules_converged(router)
self._assert_extra_routes(router)
ip_versions = [4, 6] if (ip_version == 6 or dual_stack) else [4]
self._assert_onlink_subnet_routes(router, ip_versions)
self._assert_metadata_chains(router)
# Verify router gateway interface is configured to receive Router Advts
# when IPv6 is enabled and no IPv6 gateway is configured.
if router.use_ipv6 and not v6_ext_gw_with_sub:
if not self.agent.conf.ipv6_gateway:
external_port = router.get_ex_gw_port()
external_device_name = router.get_external_device_name(
external_port['id'])
ip_wrapper = ip_lib.IPWrapper(namespace=router.ns_name)
ra_state = ip_wrapper.netns.execute(['sysctl', '-b',
'net.ipv6.conf.%s.accept_ra' % external_device_name])
self.assertEqual('2', ra_state)
if enable_ha:
self._assert_ha_device(router)
self.assertTrue(router.keepalived_manager.get_process().active)
self._delete_router(self.agent, router.router_id)
self._assert_interfaces_deleted_from_ovs()
self._assert_router_does_not_exist(router)
if enable_ha:
self.assertFalse(router.keepalived_manager.get_process().active)
def test_ipsec_site_connections_with_l3ha_routers(self):
"""Test ipsec site connection with HA routers.
This test creates two agents. First agent will have Legacy and HA
routers. Second agent will host only HA router. We setup ipsec
connection between legacy and HA router.
When HA router is created, agent1 will have master router and
agent2 will have backup router. Ipsec connection will be established
between legacy router and agent1's master HA router.
Then we fail the agent1's master HA router. Agent1's HA router will
transition to backup and agent2's HA router will become master.
Now ipsec connection will be established between legacy router and
agent2's master HA router
"""
self.failover_agent = self._configure_agent('agent2')
self.connect_agents(self.vpn_agent, self.failover_agent)
vpn_agent_driver = self.vpn_agent.device_drivers[0]
self.failover_agent_driver = self.failover_agent.device_drivers[0]
site1, site2 = self._create_ipsec_site_connection(l3ha=True)
router = site1['router']
router1 = site2['router1']
router2 = site2['router2']
# No ipsec connection between legacy router and HA routers
net_helpers.assert_no_ping(site1['port_namespace'], site2['port_ip'],
timeout=8, count=4)
net_helpers.assert_no_ping(site2['port_namespace'], site1['port_ip'],
timeout=8, count=4)
# sync the routers
vpn_agent_driver.sync(mock.Mock(), [{'id': router.router_id},
{'id': router1.router_id}])
self.failover_agent_driver.sync(mock.Mock(),
[{'id': router1.router_id}])
self.addCleanup(
vpn_agent_driver._delete_vpn_processes,
[router.router_id, router1.router_id], [])
# Test ipsec connection between legacy router and agent2's HA router
net_helpers.assert_ping(site1['port_namespace'], site2['port_ip'],
timeout=8, count=4)
net_helpers.assert_ping(site2['port_namespace'], site1['port_ip'],
timeout=8, count=4)
# Fail the agent1's HA router. Agent1's HA router will transition
# to backup and agent2's HA router will become master.
self._fail_ha_router(router1)
linux_utils.wait_until_true(lambda: router2.ha_state == 'master')
linux_utils.wait_until_true(lambda: router1.ha_state == 'backup')
# wait until ipsec process running in failover agent's HA router
# check for both strongswan and openswan processes
path = self.failover_agent_driver.processes[
router2.router_id].config_dir
pid_files = ['%s/var/run/charon.pid' % path,
'%s/var/run/pluto.pid' % path]
linux_utils.wait_until_true(
lambda: self._ipsec_process_exists(
self.failover_agent.conf, router2, pid_files))
# Test ipsec connection between legacy router and agent2's HA router
net_helpers.assert_ping(site1['port_namespace'], site2['port_ip'],
timeout=8, count=4)
net_helpers.assert_ping(site2['port_namespace'], site1['port_ip'],
timeout=8, count=4)
def wait_until_ports_state(self, ports, up, timeout=60):
port_ids = [p['id'] for p in ports]
agent_utils.wait_until_true(lambda: self._expected_plugin_rpc_call(
self.agent.plugin_rpc.update_device_list, port_ids, up),
timeout=timeout)
def block_until_boot(self):
utils.wait_until_true(lambda: (self.safe_client.client.show_port(
self.neutron_port_id)['port']['status'] == 'ACTIVE'),
sleep=3)
def test_icmp_responder(self):
"""
Add a VM. Verify the icmp flow is there.
"""
network = self.store(objects.NetworkTestObj(self.neutron, self.nb_api))
network_id = network.create(network={'name': 'icmp_responder_test'})
subnet_obj = self.store(objects.SubnetTestObj(
self.neutron,
self.nb_api,
network_id,
))
subnet = {'network_id': network_id,
'cidr': '10.10.10.0/24',
'gateway_ip': '10.10.10.1',
'ip_version': 4,
'name': 'arp_responder_test',
'enable_dhcp': True}
subnet_id = subnet_obj.create(subnet)
flows_before = self._get_l2_lookup_table_flows()
router = self.store(objects.RouterTestObj(self.neutron, self.nb_api))
router_id = router.create()
subnet_msg = {'subnet_id': subnet_id}
interface = self.neutron.add_interface_router(router_id,
body=subnet_msg)
port_id = interface['port_id']
router_port = self.neutron.list_ports(id=port_id)['ports'][0]
router_ip = None
for ip in router_port['fixed_ips']:
if ip['subnet_id'] == subnet_id:
router_ip = ip['ip_address']
break
vm = self.store(objects.VMTestObj(self, self.neutron))
vm.create(network=network)
ip = vm.get_first_ipv4()
self.assertIsNotNone(ip)
flows_middle = self._get_l2_lookup_table_flows()
vm.close()
router.close()
eventlet.sleep(test_utils.DEFAULT_CMD_TIMEOUT)
flows_delta = [flow for flow in flows_middle
if flow not in flows_before]
self.assertIsNotNone(
self._find_icmp_responder_flow_by_ip(flows_delta, router_ip)
)
condition = lambda: self._check_icmp_flow_removal(router_ip)
try:
utils.wait_until_true(
condition, timeout=40, sleep=1,
exception=RuntimeError(
"Timed out waiting for icmp responder flow from %(ip)s"
" to be removed" % {'ip': router_ip}))
except Exception as e:
self.assertIsNone(e)
finally:
network.close()
def wait_until_env_is_up(self):
utils.wait_until_true(self._processes_are_ready)
def block_until_ping(self, dst_ip):
predicate = functools.partial(self.ping_predicate, dst_ip)
utils.wait_until_true(predicate)
def wait_until_env_is_up(self, agents_count=0):
utils.wait_until_true(
functools.partial(self._processes_are_ready, agents_count))
def test_monitor_generates_initial_output(self):
utils.wait_until_true(self.collect_monitor_output, timeout=30)
def _wait_for_rule_applied_ovs_agent(vm, limit, burst):
utils.wait_until_true(
lambda: vm.bridge.get_egress_bw_limit_for_port(
vm.port.name) == (limit, burst))
def _wait_for_death(self):
is_dead = lambda: self.proc.poll() is not None
utils.wait_until_true(
is_dead, timeout=self.TIMEOUT, exception=RuntimeError(
"Ping command hasn't ended after %d seconds." % self.TIMEOUT))