Ejemplo n.º 1
0
def test_icmp_failback():
    """
    Title: BGP failover/failback

    Scenario 1:
    Given: multiple uplinks
    When: enable BGP on both of them
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 2:
    Given:
    When: inject failure into one of BGP uplinks (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 3:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 4:
    Given:
    When: inject failure into another of BGP uplinks (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 5:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a pseudo public IP address

    """
    add_bgp([uplink1_session1], route_direct)
    add_bgp([uplink2_session1], route_direct)

    ping_to_inet() # BGP #1 and #2 are working

    failure = PktFailure('quagga1', 'bgp1', 5)

    failure.inject()
    try:
        ping_to_inet() # BGP #1 is lost but continues to work
    finally:
        failure.eject()

    await_internal_route_exported(64513, 64511)
    ping_to_inet() # BGP #1 is back

    failure = PktFailure('quagga2', 'bgp1', 5)
    failure.inject()
    try:
        ping_to_inet() # BGP #2 is lost but continues to work
    except Exception as e:
        raise e
    finally:
        failure.eject()

    await_internal_route_exported(64513, 64512)
    ping_to_inet()  # BGP #2 is back
Ejemplo n.º 2
0
def test_icmp_failback():
    """
    Title: BGP failover/failback

    Scenario 1:
    Given: multiple uplinks
    When: enable BGP on both of them
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 2:
    Given:
    When: inject failure into one of BGP uplinks (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 3:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 4:
    Given:
    When: inject failure into another of BGP uplinks (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 5:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a pseudo public IP address

    """
    add_bgp_1(route_direct)
    add_bgp_2(route_direct)

    ping_inet() # BGP #1 and #2 are working

    failure = PktFailure('quagga', 'bgp0', 15)

    failure.inject()
    try:
        ping_inet() # BGP #1 is lost
    finally:
        failure.eject()

    ping_inet() # BGP #1 is back

    failure = PktFailure('quagga', 'bgp1', 15)
    failure.inject()
    try:
        ping_inet() # BGP #2 is lost
    finally:
        failure.eject()

    ping_inet()  # BGP #2 is back

    clear_bgp()
Ejemplo n.º 3
0
def test_multisession_icmp_failback():
    """
    Title: BGP session failover/failback

    Scenario 1:
    Given: single uplink with multiple sessions
    When: enable two bgp sessions on the same port
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 2:
    Given:
    When: inject failure into one of BGP session (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 3:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 4:
    Given:
    When: inject failure into the other BGP session (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 5:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a pseudo public IP address

    """
    add_bgp(uplink1_multisession, route_direct)
    ping_to_inet() # BGP #1 and #2 are working

    failure = PktFailure('quagga1', 'bgp1', 5)
    failure.inject()
    try:
        ping_to_inet() # BGP session #1 is lost but continues to work
    except Exception as e:
        raise e
    finally:
        failure.eject()

    ping_to_inet() # BGP #1 is back

    failure = PktFailure('quagga2', 'bgp2', 5)
    failure.inject()
    try:
        ping_to_inet() # BGP session #2 is lost but continues to work
    finally:
        failure.eject()

    ping_to_inet()  # BGP #2 is back
Ejemplo n.º 4
0
def test_multisession_icmp_with_redundancy():
    """
    Title: BGP adding double session redundancy to two uplinks

    Scenario:
    Given: two uplinks with two sessions each
    When: disabling one by one
    Then: ICMP echo should work from different vms to a pseudo public IP address
    """
    add_bgp(uplink1_multisession, route_direct)

    add_bgp(uplink2_multisession, route_direct)

    failures = []

    ping_to_inet() # Midolman 2 with 2 sessions on uplink & uplink2

    # Start failing sessions one by one
    failure1 = PktFailure('quagga1', 'bgp1', 5)
    failures.append(failure1)
    failure1.inject()
    try:
        ping_to_inet()
    except:
        for failure in failures:
            failure.eject()
        raise

    failure2 = PktFailure('quagga2', 'bgp1', 5)
    failures.append(failure2)
    failure2.inject()
    try:
        ping_to_inet()
    except:
        for failure in failures:
            failure.eject()
        raise

    failure3 = PktFailure('quagga1', 'bgp2', 5)
    failures.append(failure3)
    failure3.inject()
    try:
        ping_to_inet()
    except:
        for failure in failures:
            failure.eject()
        raise

    failure4 = PktFailure('quagga2', 'bgp2', 5)
    failures.append(failure4)
    failure4.inject()
    assert_that(calling(ping_to_inet),
                raises(RuntimeError))

    for failure in failures:
        failure.eject()
Ejemplo n.º 5
0
def test_icmp_failback():
    """
    Title: BGP failover/failback

    Scenario 1:
    Given: multiple uplinks
    When: enable BGP on both of them
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 2:
    Given:
    When: inject failure into one of BGP uplinks (failover)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 3:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 4:
    Given:
    When: inject failure into another of BGP uplinks (failover)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 5:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a psudo public IP address

    """
    (p1, p2) = (add_bgp_1(route_direct), add_bgp_2(route_direct))

    ping_inet()  # BGP #1 and #2 are working

    failure = PktFailure(NS_BGP_PEER_1, 'eth0', 30)
    failure.inject()
    try:
        ping_inet()  # BGP #1 is lost
    finally:
        failure.eject()

    ping_inet()  # BGP #1 is back

    failure = PktFailure(NS_BGP_PEER_1, 'eth1', 30)
    failure.inject()
    try:
        ping_inet()  # BGP #2 is lost
    finally:
        failure.eject()

    ping_inet()  # BGP #2 is back

    clear_bgp(p1)
    clear_bgp(p2)
Ejemplo n.º 6
0
def test_multisession_icmp_with_redundancy():
    """
    Title: BGP adding double session redundancy to two uplinks

    Scenario:
    Given: two uplinks with two sessions each
    When: disabling one by one
    Then: ICMP echo should work from different vms to a pseudo public IP address
    """
    add_bgp(uplink1_multisession, route_direct)

    add_bgp(uplink2_multisession, route_direct)

    failures = []

    ping_to_inet()  # Midolman 2 with 2 sessions on uplink & uplink2

    # Start failing sessions one by one
    failure1 = PktFailure('quagga1', 'bgp1', 5)
    failures.append(failure1)
    failure1.inject()
    try:
        ping_to_inet()
    except:
        for failure in failures:
            failure.eject()
        raise

    failure2 = PktFailure('quagga2', 'bgp1', 5)
    failures.append(failure2)
    failure2.inject()
    try:
        ping_to_inet()
    except:
        for failure in failures:
            failure.eject()
        raise

    failure3 = PktFailure('quagga1', 'bgp2', 5)
    failures.append(failure3)
    failure3.inject()
    try:
        ping_to_inet()
    except:
        for failure in failures:
            failure.eject()
        raise

    failure4 = PktFailure('quagga2', 'bgp2', 5)
    failures.append(failure4)
    failure4.inject()
    assert_that(calling(ping_to_inet), raises(RuntimeError))

    for failure in failures:
        failure.eject()
Ejemplo n.º 7
0
def test_multisession_icmp_failback():
    """
    Title: BGP session failover/failback

    Scenario 1:
    Given: single uplink with multiple sessions
    When: enable two bgp sessions on the same port
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 2:
    Given:
    When: inject failure into one of BGP session (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 3:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 4:
    Given:
    When: inject failure into the other BGP session (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 5:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a pseudo public IP address

    """
    add_bgp(uplink1_multisession, route_direct)
    ping_to_inet()  # BGP #1 and #2 are working

    failure = PktFailure('quagga1', 'bgp1', 5)
    failure.inject()
    try:
        ping_to_inet()  # BGP session #1 is lost but continues to work
    finally:
        failure.eject()

    ping_to_inet()  # BGP #1 is back

    failure = PktFailure('quagga2', 'bgp2', 5)
    failure.inject()
    try:
        ping_to_inet()  # BGP session #2 is lost but continues to work
    finally:
        failure.eject()

    ping_to_inet()  # BGP #2 is back
Ejemplo n.º 8
0
def test_icmp_failback():
    """
    Title: BGP failover/failback

    Scenario 1:
    Given: multiple uplinks
    When: enable BGP on both of them
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 2:
    Given:
    When: inject failure into one of BGP uplinks (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 3:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 4:
    Given:
    When: inject failure into another of BGP uplinks (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 5:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a pseudo public IP address

    """
    add_bgp([uplink1_session1], route_direct)
    add_bgp([uplink2_session1], route_direct)

    ping_to_inet()  # BGP #1 and #2 are working

    failure = PktFailure('quagga1', 'bgp1', 5)

    failure.inject()
    try:
        ping_to_inet()  # BGP #1 is lost but continues to work
    finally:
        failure.eject()

    await_internal_route_exported(64513, 64511)
    ping_to_inet()  # BGP #1 is back

    failure = PktFailure('quagga2', 'bgp1', 5)
    failure.inject()
    try:
        ping_to_inet()  # BGP #2 is lost but continues to work
    finally:
        failure.eject()

    await_internal_route_exported(64513, 64512)
    ping_to_inet()  # BGP #2 is back
Ejemplo n.º 9
0
def test_multisession_icmp_with_redundancy():
    """
    Title: BGP adding double session redundancy to two uplinks

    Scenario:
    Given: two uplinks with two sessions each
    When: disabling one by one
    Then: ICMP echo should work from different vms to a pseudo public IP address
    """
    def await_routes(session, exists=True):
        await_router_route(session, '1.1.1.1', 32, exists)
        await_router_route(session, '0.0.0.0', 0, exists)

    add_bgp(uplink1_multisession, route_direct)
    await_routes(uplink1_session1, exists=True)
    await_routes(uplink1_session2, exists=True)

    add_bgp(uplink2_multisession, route_direct)
    await_routes(uplink2_session1, exists=True)
    await_routes(uplink2_session2, exists=True)

    failures = []

    ping_to_inet()  # Midolman 2 with 2 sessions on uplink & uplink2

    # Start failing sessions one by one
    failure1 = PktFailure(uplink1_session1['peerHostname'],
                          uplink1_session1['peerIface'], 5)
    failures.append(failure1)
    failure1.inject()
    try:
        await_routes(uplink1_session1, exists=False)
        ping_to_inet()
    except Exception:
        for failure in failures:
            failure.eject()
        raise

    failure2 = PktFailure(uplink2_session1['peerHostname'],
                          uplink2_session1['peerIface'], 5)
    failures.append(failure2)
    failure2.inject()
    try:
        await_routes(uplink2_session1, exists=False)
        ping_to_inet()
    except Exception:
        for failure in failures:
            failure.eject()
        raise

    failure3 = PktFailure(uplink1_session2['peerHostname'],
                          uplink1_session2['peerIface'], 5)
    failures.append(failure3)
    failure3.inject()
    try:
        await_routes(uplink1_session2, exists=False)
        ping_to_inet()
    except Exception:
        for failure in failures:
            failure.eject()
        raise

    failure4 = PktFailure(uplink2_session2['peerHostname'],
                          uplink2_session2['peerIface'], 5)
    failures.append(failure4)
    failure4.inject()
    try:
        await_routes(uplink2_session2, exists=False)
    except Exception:
        for failure in failures:
            failure.eject()
        raise

    try:
        assert_that(calling(ping_to_inet), raises(RuntimeError))
    finally:
        for failure in failures:
            failure.eject()
Ejemplo n.º 10
0
def test_multisession_icmp_failback():
    """
    Title: BGP session failover/failback

    Scenario 1:
    Given: single uplink with multiple sessions
    When: enable two bgp sessions on the same port
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 2:
    Given:
    When: inject failure into one of BGP session (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 3:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 4:
    Given:
    When: inject failure into the other BGP session (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 5:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a pseudo public IP address

    """
    data = BM.get_binding_data()['config']
    session1 = data['session1']
    session2 = data['session2']
    multisession = BM.get_binding_data()['config']['multisession']

    add_bgp(multisession, route_direct)

    ping_to_inet()  # BGP #1 and #2 are working

    failure1 = PktFailure(session1['peerHostname'], session1['peerIface'], 5)
    failure1.inject()
    try:
        ping_to_inet()  # BGP session #1 is lost but continues to work
    finally:
        failure1.eject()

    ping_to_inet()  # BGP #1 is back

    failure2 = PktFailure(session2['peerHostname'], session2['peerIface'], 5)
    failure2.inject()
    try:
        ping_to_inet()  # BGP session #2 is lost but continues to work
    finally:
        failure2.eject()

    ping_to_inet()  # BGP #2 is back

    # Check routes
    await_router_route(session1, '1.1.1.1', 32, exists=True)
    await_router_route(session1, '0.0.0.0', 0, exists=True)
    await_router_route(session2, '1.1.1.1', 32, exists=True)
    await_router_route(session2, '0.0.0.0', 0, exists=True)

    # Inject failures again and check routes
    failure1.inject()
    try:
        await_router_route(session1, '1.1.1.1', 32, exists=False)
        await_router_route(session1, '0.0.0.0', 0, exists=False)
        await_router_route(session2, '1.1.1.1', 32, exists=True)
        await_router_route(session2, '0.0.0.0', 0, exists=True)
    except Exception:
        failure1.eject()
        raise

    failure2.inject()
    try:
        await_router_route(session1, '1.1.1.1', 32, exists=False)
        await_router_route(session1, '0.0.0.0', 0, exists=False)
        await_router_route(session2, '1.1.1.1', 32, exists=False)
        await_router_route(session2, '0.0.0.0', 0, exists=False)
    except Exception:
        failure2.eject()
        raise

    failure1.eject()
    failure2.eject()
Ejemplo n.º 11
0
def test_icmp_failback():
    """
    Title: BGP failover/failback

    Scenario 1:
    Given: multiple uplinks
    When: enable BGP on both of them
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 2:
    Given:
    When: inject failure into one of BGP uplinks (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 3:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 4:
    Given:
    When: inject failure into another of BGP uplinks (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 5:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a pseudo public IP address

    """
    add_bgp([uplink1_session1], route_direct)
    add_bgp([uplink2_session1], route_direct)

    await_router_route(uplink1_session1, '1.1.1.1', 32, exists=True)
    await_router_route(uplink1_session1, '0.0.0.0', 0, exists=True)
    await_router_route(uplink2_session1, '1.1.1.1', 32, exists=True)
    await_router_route(uplink2_session1, '0.0.0.0', 0, exists=True)

    ping_to_inet()  # BGP #1 and #2 are working

    failure1 = PktFailure(uplink1_session1['peerHostname'],
                          uplink1_session1['peerIface'], 5)

    failure1.inject()
    try:
        await_router_route(uplink1_session1, '1.1.1.1', 32, exists=False)
        await_router_route(uplink1_session1, '0.0.0.0', 0, exists=False)
        await_router_route(uplink2_session1, '1.1.1.1', 32, exists=True)
        await_router_route(uplink2_session1, '0.0.0.0', 0, exists=True)
        ping_to_inet()  # BGP #1 is lost but continues to work
    finally:
        failure1.eject()

    await_internal_route_exported(64513, 64511)
    ping_to_inet()  # BGP #1 is back

    failure2 = PktFailure(uplink2_session1['peerHostname'],
                          uplink2_session1['peerIface'], 5)
    failure2.inject()
    try:
        await_router_route(uplink1_session1, '1.1.1.1', 32, exists=True)
        await_router_route(uplink1_session1, '0.0.0.0', 0, exists=True)
        await_router_route(uplink2_session1, '1.1.1.1', 32, exists=False)
        await_router_route(uplink2_session1, '0.0.0.0', 0, exists=False)
        ping_to_inet()  # BGP #2 is lost but continues to work
    finally:
        failure2.eject()

    await_internal_route_exported(64513, 64512)
    ping_to_inet()  # BGP #2 is back

    failure1.inject()
    failure2.inject()
    try:
        await_router_route(uplink1_session1, '1.1.1.1', 32, exists=False)
        await_router_route(uplink1_session1, '0.0.0.0', 0, exists=False)
        await_router_route(uplink2_session1, '1.1.1.1', 32, exists=False)
        await_router_route(uplink2_session1, '0.0.0.0', 0, exists=False)
    finally:
        failure1.eject()
        failure2.eject()
Ejemplo n.º 12
0
def test_multisession_icmp_with_redundancy():
    """
    Title: BGP adding double session redundancy to two uplinks

    Scenario:
    Given: two uplinks with two sessions each
    When: disabling one by one
    Then: ICMP echo should work from different vms to a pseudo public IP address
    """

    def await_routes(session, exists=True):
        await_router_route(session, '1.1.1.1', 32, exists)
        await_router_route(session, '0.0.0.0', 0,  exists)

    add_bgp(uplink1_multisession, route_direct)
    await_routes(uplink1_session1, exists=True)
    await_routes(uplink1_session2, exists=True)

    add_bgp(uplink2_multisession, route_direct)
    await_routes(uplink2_session1, exists=True)
    await_routes(uplink2_session2, exists=True)

    failures = []

    ping_to_inet()  # Midolman 2 with 2 sessions on uplink & uplink2

    # Start failing sessions one by one
    failure1 = PktFailure(uplink1_session1['peerHostname'],
                          uplink1_session1['peerIface'], 5)
    failures.append(failure1)
    failure1.inject()
    try:
        await_routes(uplink1_session1, exists=False)
        ping_to_inet()
    except Exception:
        for failure in failures:
            failure.eject()
        raise

    failure2 = PktFailure(uplink2_session1['peerHostname'],
                          uplink2_session1['peerIface'], 5)
    failures.append(failure2)
    failure2.inject()
    try:
        await_routes(uplink2_session1, exists=False)
        ping_to_inet()
    except Exception:
        for failure in failures:
            failure.eject()
        raise

    failure3 = PktFailure(uplink1_session2['peerHostname'],
                          uplink1_session2['peerIface'], 5)
    failures.append(failure3)
    failure3.inject()
    try:
        await_routes(uplink1_session2, exists=False)
        ping_to_inet()
    except Exception:
        for failure in failures:
            failure.eject()
        raise

    failure4 = PktFailure(uplink2_session2['peerHostname'],
                          uplink2_session2['peerIface'], 5)
    failures.append(failure4)
    failure4.inject()
    try:
        await_routes(uplink2_session2, exists=False)
    except Exception:
        for failure in failures:
            failure.eject()
        raise

    try:
        assert_that(calling(ping_to_inet),
                    raises(RuntimeError))
    finally:
        for failure in failures:
            failure.eject()
Ejemplo n.º 13
0
def test_multisession_icmp_failback():
    """
    Title: BGP session failover/failback

    Scenario 1:
    Given: single uplink with multiple sessions
    When: enable two bgp sessions on the same port
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 2:
    Given:
    When: inject failure into one of BGP session (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 3:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 4:
    Given:
    When: inject failure into the other BGP session (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 5:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a pseudo public IP address

    """
    data = BM.get_binding_data()['config']
    session1 = data['session1']
    session2 = data['session2']
    multisession = BM.get_binding_data()['config']['multisession']

    add_bgp(multisession, route_direct)

    ping_to_inet()  # BGP #1 and #2 are working

    failure1 = PktFailure(session1['peerHostname'], session1['peerIface'], 5)
    failure1.inject()
    try:
        ping_to_inet()  # BGP session #1 is lost but continues to work
    finally:
        failure1.eject()

    ping_to_inet()  # BGP #1 is back

    failure2 = PktFailure(session2['peerHostname'], session2['peerIface'], 5)
    failure2.inject()
    try:
        ping_to_inet()  # BGP session #2 is lost but continues to work
    finally:
        failure2.eject()

    ping_to_inet()  # BGP #2 is back

    # Check routes
    await_router_route(session1, '1.1.1.1', 32, exists=True)
    await_router_route(session1, '0.0.0.0', 0,  exists=True)
    await_router_route(session2, '1.1.1.1', 32, exists=True)
    await_router_route(session2, '0.0.0.0', 0,  exists=True)

    # Inject failures again and check routes
    failure1.inject()
    try:
        await_router_route(session1, '1.1.1.1', 32, exists=False)
        await_router_route(session1, '0.0.0.0', 0,  exists=False)
        await_router_route(session2, '1.1.1.1', 32, exists=True)
        await_router_route(session2, '0.0.0.0', 0,  exists=True)
    except Exception:
        failure1.eject()
        raise

    failure2.inject()
    try:
        await_router_route(session1, '1.1.1.1', 32, exists=False)
        await_router_route(session1, '0.0.0.0', 0,  exists=False)
        await_router_route(session2, '1.1.1.1', 32, exists=False)
        await_router_route(session2, '0.0.0.0', 0,  exists=False)
    except Exception:
        failure2.eject()
        raise

    failure1.eject()
    failure2.eject()
Ejemplo n.º 14
0
def test_icmp_failback():
    """
    Title: BGP failover/failback

    Scenario 1:
    Given: multiple uplinks
    When: enable BGP on both of them
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 2:
    Given:
    When: inject failure into one of BGP uplinks (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 3:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a psudo public IP address

    Scenario 4:
    Given:
    When: inject failure into another of BGP uplinks (failover)
    Then: ICMP echo RR should work to a pseudo public IP address

    Scenario 5:
    Given:
    When: eject failure (failback)
    Then: ICMP echo RR should work to a pseudo public IP address

    """
    add_bgp([uplink1_session1], route_direct)
    add_bgp([uplink2_session1], route_direct)

    await_router_route(uplink1_session1, '1.1.1.1', 32, exists=True)
    await_router_route(uplink1_session1, '0.0.0.0', 0,  exists=True)
    await_router_route(uplink2_session1, '1.1.1.1', 32, exists=True)
    await_router_route(uplink2_session1, '0.0.0.0', 0,  exists=True)

    ping_to_inet()  # BGP #1 and #2 are working

    failure1 = PktFailure(uplink1_session1['peerHostname'],
                          uplink1_session1['peerIface'], 5)

    failure1.inject()
    try:
        await_router_route(uplink1_session1, '1.1.1.1', 32, exists=False)
        await_router_route(uplink1_session1, '0.0.0.0', 0,  exists=False)
        await_router_route(uplink2_session1, '1.1.1.1', 32, exists=True)
        await_router_route(uplink2_session1, '0.0.0.0', 0,  exists=True)
        ping_to_inet()  # BGP #1 is lost but continues to work
    finally:
        failure1.eject()

    await_internal_route_exported(64513, 64511)
    ping_to_inet()  # BGP #1 is back

    failure2 = PktFailure(uplink2_session1['peerHostname'],
                          uplink2_session1['peerIface'], 5)
    failure2.inject()
    try:
        await_router_route(uplink1_session1, '1.1.1.1', 32, exists=True)
        await_router_route(uplink1_session1, '0.0.0.0', 0,  exists=True)
        await_router_route(uplink2_session1, '1.1.1.1', 32, exists=False)
        await_router_route(uplink2_session1, '0.0.0.0', 0,  exists=False)
        ping_to_inet()  # BGP #2 is lost but continues to work
    finally:
        failure2.eject()

    await_internal_route_exported(64513, 64512)
    ping_to_inet()  # BGP #2 is back

    failure1.inject()
    failure2.inject()
    try:
        await_router_route(uplink1_session1, '1.1.1.1', 32, exists=False)
        await_router_route(uplink1_session1, '0.0.0.0', 0,  exists=False)
        await_router_route(uplink2_session1, '1.1.1.1', 32, exists=False)
        await_router_route(uplink2_session1, '0.0.0.0', 0,  exists=False)
    finally:
        failure1.eject()
        failure2.eject()