def setUp(self):
super(TestEtcdStatusReporter, self).setUp()
self.m_config = Mock(spec=Config)
self.m_config.ETCD_ADDR = ETCD_ADDRESS
self.m_config.HOSTNAME = "foo"
self.m_config.REPORT_ENDPOINT_STATUS = True
self.m_config.ENDPOINT_REPORT_DELAY = 1
self.m_client = Mock()
self.rep = EtcdStatusReporter(self.m_config)
self.rep.client = self.m_client
def setUp(self):
super(TestEtcdStatusReporter, self).setUp()
self.m_config = Mock(spec=Config)
self.m_config.ETCD_ADDRS = [ETCD_ADDRESS]
self.m_config.ETCD_SCHEME = "http"
self.m_config.ETCD_KEY_FILE = None
self.m_config.ETCD_CERT_FILE = None
self.m_config.ETCD_CA_FILE = None
self.m_config.HOSTNAME = "foo"
self.m_config.REPORT_ENDPOINT_STATUS = True
self.m_config.ENDPOINT_REPORT_DELAY = 1
self.m_client = Mock()
self.rep = EtcdStatusReporter(self.m_config)
self.rep.client = self.m_client
class TestEtcdStatusReporter(BaseTestCase):
def setUp(self):
super(TestEtcdStatusReporter, self).setUp()
self.m_config = Mock(spec=Config)
self.m_config.ETCD_ADDRS = [ETCD_ADDRESS]
self.m_config.ETCD_SCHEME = "http"
self.m_config.ETCD_KEY_FILE = None
self.m_config.ETCD_CERT_FILE = None
self.m_config.ETCD_CA_FILE = None
self.m_config.HOSTNAME = "foo"
self.m_config.REPORT_ENDPOINT_STATUS = True
self.m_config.ENDPOINT_REPORT_DELAY = 1
self.m_client = Mock()
self.rep = EtcdStatusReporter(self.m_config)
self.rep.client = self.m_client
def test_on_endpoint_status_mainline(self):
# Send in an endpoint status update.
endpoint_id = WloadEndpointId("foo", "bar", "baz", "biff")
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep.on_endpoint_status_changed(endpoint_id, IPV4,
{"status": "up"},
async=True)
self.step_actor(self.rep)
# Should record the status.
self.assertEqual(
self.rep._endpoint_status[IPV4],
{
endpoint_id: {"status": "up"}
}
)
# And do a write.
self.assertEqual(
self.m_client.set.mock_calls,
[call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff",
JSONString({"status": "up"}))]
)
# Since we did a write, the rate limit timer should be scheduled.
self.assertEqual(
m_spawn.mock_calls,
[call(ANY, self.rep._on_timer_pop, async=True)]
)
self.assertTrue(self.rep._timer_scheduled)
self.assertFalse(self.rep._reporting_allowed)
# Send in another update, shouldn't get written until we pop the timer.
self.m_client.reset_mock()
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep.on_endpoint_status_changed(endpoint_id,
IPV4,
None,
async=True)
self.step_actor(self.rep)
self.assertFalse(self.m_client.set.called)
# Timer already scheduled, shouldn't get rescheduled.
self.assertFalse(m_spawn.called)
# Pop the timer, should trigger write and reschedule.
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep._on_timer_pop(async=True)
self.step_actor(self.rep)
self.maxDiff = 10000
self.assertEqual(
self.m_client.delete.mock_calls,
[
call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/"
"biff"),
call("calico/felix/v1/host/foo/workload/bar/baz/endpoint",
dir=True, timeout=5),
call("calico/felix/v1/host/foo/workload/bar/baz",
dir=True, timeout=5),
call("calico/felix/v1/host/foo/workload/bar",
dir=True, timeout=5),
call("calico/felix/v1/host/foo/workload",
dir=True, timeout=5),
]
)
# Rate limit timer should be scheduled.
self.assertEqual(
m_spawn.mock_calls,
[call(ANY, self.rep._on_timer_pop, async=True)]
)
spawn_delay = m_spawn.call_args[0][0]
self.assertTrue(spawn_delay >= 0.89999)
self.assertTrue(spawn_delay <= 1.10001)
self.assertTrue(self.rep._timer_scheduled)
self.assertFalse(self.rep._reporting_allowed)
# Cache should be cleaned up.
self.assertEqual(self.rep._endpoint_status[IPV4], {})
# Nothing queued.
self.assertEqual(self.rep._newer_dirty_endpoints, set())
self.assertEqual(self.rep._older_dirty_endpoints, set())
def test_mark_endpoint_dirty_already_dirty(self):
endpoint_id = WloadEndpointId("a", "b", "c", "d")
self.rep._older_dirty_endpoints.add(endpoint_id)
self.rep._mark_endpoint_dirty(endpoint_id)
self.assertFalse(endpoint_id in self.rep._newer_dirty_endpoints)
def test_on_endpoint_status_failure(self):
# Send in an endpoint status update.
endpoint_id = WloadEndpointId("foo", "bar", "baz", "biff")
self.m_client.set.side_effect = EtcdException()
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep.on_endpoint_status_changed(endpoint_id,
IPV4,
{"status": "up"},
async=True)
self.step_actor(self.rep)
# Should do the write.
self.assertEqual(
self.m_client.set.mock_calls,
[call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff",
JSONString({"status": "up"}))]
)
# But endpoint should be re-queued in the newer set.
self.assertEqual(self.rep._newer_dirty_endpoints, set([endpoint_id]))
self.assertEqual(self.rep._older_dirty_endpoints, set())
def test_on_endpoint_status_changed_disabled(self):
self.m_config.REPORT_ENDPOINT_STATUS = False
endpoint_id = WloadEndpointId("foo", "bar", "baz", "biff")
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep.on_endpoint_status_changed(endpoint_id,
IPV4,
{"status": "up"},
async=True)
self.step_actor(self.rep)
self.assertFalse(m_spawn.called)
self.assertEqual(self.rep._endpoint_status[IPV4], {})
# Nothing queued.
self.assertEqual(self.rep._newer_dirty_endpoints, set())
self.assertEqual(self.rep._older_dirty_endpoints, set())
def test_on_endpoint_status_v4_v6(self):
# Send in endpoint status updates for v4 and v6.
endpoint_id = WloadEndpointId("foo", "bar", "baz", "biff")
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep.on_endpoint_status_changed(endpoint_id, IPV4,
{"status": "up"},
async=True)
self.rep.on_endpoint_status_changed(endpoint_id, IPV6,
{"status": "down"},
async=True)
self.step_actor(self.rep)
# Should record the status.
self.assertEqual(
self.rep._endpoint_status,
{
IPV4: {endpoint_id: {"status": "up"}},
IPV6: {endpoint_id: {"status": "down"}},
}
)
# And do a write.
self.assertEqual(
self.m_client.set.mock_calls,
[call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff",
JSONString({"status": "down"}))]
)
def test_resync(self):
endpoint_id = WloadEndpointId("foo", "bar", "baz", "biff")
self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True)
endpoint_id_2 = WloadEndpointId("foo", "bar", "baz", "boff")
self.rep.on_endpoint_status_changed(endpoint_id_2, IPV6, {"status": "up"}, async=True)
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.step_actor(self.rep)
self.rep._on_timer_pop(async=True)
self.step_actor(self.rep)
self.assertEqual(self.rep._older_dirty_endpoints, set())
self.assertEqual(self.rep._newer_dirty_endpoints, set())
self.rep.resync(async=True)
self.step_actor(self.rep)
self.assertEqual(self.rep._older_dirty_endpoints, set())
self.assertEqual(self.rep._newer_dirty_endpoints, set([endpoint_id, endpoint_id_2]))
def test_combine_statuses(self):
"""
Test the "truth table" for combining status reports.
"""
self.assert_combined_status(None, None, None)
self.assert_combined_status({"status": "up"}, None, {"status": "up"})
self.assert_combined_status({"status": "up"}, {"status": "up"},
{"status": "up"})
self.assert_combined_status({"status": "down"}, {"status": "up"},
{"status": "down"})
self.assert_combined_status({"status": "error"}, {"status": "up"},
{"status": "error"})
def assert_combined_status(self, a, b, expected):
# Should be symmetric so check the arguments both ways round.
for lhs, rhs in [(a, b), (b, a)]:
result = combine_statuses(lhs, rhs)
self.assertEqual(result, expected,
"Expected %r and %r to combine to %s but got %r" %
(lhs, rhs, expected, result))
def test_clean_up_endpoint_status(self):
self.m_config.REPORT_ENDPOINT_STATUS = True
ep_id = WloadEndpointId("foo",
"openstack",
"workloadid",
"endpointid")
empty_dir = Mock()
empty_dir.key = ("/calico/felix/v1/host/foo/workload/"
"openstack/foobar")
empty_dir.dir = True
missing_ep = Mock()
missing_ep.key = ("/calico/felix/v1/host/foo/workload/"
"openstack/aworkload/endpoint/anendpoint")
self.m_client.read.return_value.leaves = [
empty_dir,
missing_ep,
]
with patch.object(self.rep, "_mark_endpoint_dirty") as m_mark:
self.rep.clean_up_endpoint_statuses(async=True)
self.step_actor(self.rep)
# Missing endpoint should have been marked for cleanup.
m_mark.assert_called_once_with(
WloadEndpointId("foo",
"openstack",
"aworkload",
"anendpoint")
)
def test_clean_up_endpoint_status_etcd_error(self):
self.m_config.REPORT_ENDPOINT_STATUS = True
with patch.object(self.rep, "_attempt_cleanup") as m_clean:
m_clean.side_effect = EtcdException()
self.rep.clean_up_endpoint_statuses(async=True)
self.step_actor(self.rep)
self.assertTrue(self.rep._cleanup_pending)
def test_clean_up_endpoint_status_not_found(self):
self.m_config.REPORT_ENDPOINT_STATUS = True
self.m_client.read.side_effect = etcd.EtcdKeyNotFound()
with patch.object(self.rep, "_mark_endpoint_dirty") as m_mark:
self.rep.clean_up_endpoint_statuses(async=True)
self.step_actor(self.rep)
self.assertFalse(m_mark.called)
def test_clean_up_endpoint_status_disabled(self):
self.m_config.REPORT_ENDPOINT_STATUS = False
self.m_client.read.side_effect = self.failureException
self.rep.clean_up_endpoint_statuses(async=True)
self.step_actor(self.rep)
class TestEtcdStatusReporter(BaseTestCase):
def setUp(self):
super(TestEtcdStatusReporter, self).setUp()
self.m_config = Mock(spec=Config)
self.m_config.ETCD_ADDR = ETCD_ADDRESS
self.m_config.HOSTNAME = "foo"
self.m_config.REPORT_ENDPOINT_STATUS = True
self.m_config.ENDPOINT_REPORT_DELAY = 1
self.m_client = Mock()
self.rep = EtcdStatusReporter(self.m_config)
self.rep.client = self.m_client
def test_on_endpoint_status_mainline(self):
# Send in an endpoint status update.
endpoint_id = EndpointId("foo", "bar", "baz", "biff")
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep.on_endpoint_status_changed(endpoint_id, IPV4,
{"status": "up"},
async=True)
self.step_actor(self.rep)
# Should record the status.
self.assertEqual(
self.rep._endpoint_status[IPV4],
{
endpoint_id: {"status": "up"}
}
)
# And do a write.
self.assertEqual(
self.m_client.set.mock_calls,
[call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff",
JSONString({"status": "up"}))]
)
# Since we did a write, the rate limit timer should be scheduled.
self.assertEqual(
m_spawn.mock_calls,
[call(ANY, self.rep._on_timer_pop, async=True)]
)
self.assertTrue(self.rep._timer_scheduled)
self.assertFalse(self.rep._reporting_allowed)
# Send in another update, shouldn't get written until we pop the timer.
self.m_client.reset_mock()
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep.on_endpoint_status_changed(endpoint_id,
IPV4,
None,
async=True)
self.step_actor(self.rep)
self.assertFalse(self.m_client.set.called)
# Timer already scheduled, shouldn't get rescheduled.
self.assertFalse(m_spawn.called)
# Pop the timer, should trigger write and reschedule.
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep._on_timer_pop(async=True)
self.step_actor(self.rep)
self.maxDiff = 10000
self.assertEqual(
self.m_client.delete.mock_calls,
[
call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/"
"biff"),
call("calico/felix/v1/host/foo/workload/bar/baz/endpoint",
dir=True, timeout=5),
call("calico/felix/v1/host/foo/workload/bar/baz",
dir=True, timeout=5),
call("calico/felix/v1/host/foo/workload/bar",
dir=True, timeout=5),
call("calico/felix/v1/host/foo/workload",
dir=True, timeout=5),
]
)
# Rate limit timer should be scheduled.
self.assertEqual(
m_spawn.mock_calls,
[call(ANY, self.rep._on_timer_pop, async=True)]
)
spawn_delay = m_spawn.call_args[0][0]
self.assertTrue(spawn_delay >= 0.89999)
self.assertTrue(spawn_delay <= 1.10001)
self.assertTrue(self.rep._timer_scheduled)
self.assertFalse(self.rep._reporting_allowed)
# Cache should be cleaned up.
self.assertEqual(self.rep._endpoint_status[IPV4], {})
# Nothing queued.
self.assertEqual(self.rep._newer_dirty_endpoints, set())
self.assertEqual(self.rep._older_dirty_endpoints, set())
def test_on_endpoint_status_failure(self):
# Send in an endpoint status update.
endpoint_id = EndpointId("foo", "bar", "baz", "biff")
self.m_client.set.side_effect = EtcdException()
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep.on_endpoint_status_changed(endpoint_id,
IPV4,
{"status": "up"},
async=True)
self.step_actor(self.rep)
# Should do the write.
self.assertEqual(
self.m_client.set.mock_calls,
[call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff",
JSONString({"status": "up"}))]
)
# But endpoint should be re-queued in the newer set.
self.assertEqual(self.rep._newer_dirty_endpoints, set([endpoint_id]))
self.assertEqual(self.rep._older_dirty_endpoints, set())
def test_on_endpoint_status_changed_disabled(self):
self.m_config.REPORT_ENDPOINT_STATUS = False
endpoint_id = EndpointId("foo", "bar", "baz", "biff")
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep.on_endpoint_status_changed(endpoint_id,
IPV4,
{"status": "up"},
async=True)
self.step_actor(self.rep)
self.assertFalse(m_spawn.called)
self.assertEqual(self.rep._endpoint_status[IPV4], {})
# Nothing queued.
self.assertEqual(self.rep._newer_dirty_endpoints, set())
self.assertEqual(self.rep._older_dirty_endpoints, set())
def test_on_endpoint_status_v4_v6(self):
# Send in endpoint status updates for v4 and v6.
endpoint_id = EndpointId("foo", "bar", "baz", "biff")
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.rep.on_endpoint_status_changed(endpoint_id, IPV4,
{"status": "up"},
async=True)
self.rep.on_endpoint_status_changed(endpoint_id, IPV6,
{"status": "down"},
async=True)
self.step_actor(self.rep)
# Should record the status.
self.assertEqual(
self.rep._endpoint_status,
{
IPV4: {endpoint_id: {"status": "up"}},
IPV6: {endpoint_id: {"status": "down"}},
}
)
# And do a write.
self.assertEqual(
self.m_client.set.mock_calls,
[call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff",
JSONString({"status": "down"}))]
)
def test_resync(self):
endpoint_id = EndpointId("foo", "bar", "baz", "biff")
self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True)
endpoint_id_2 = EndpointId("foo", "bar", "baz", "boff")
self.rep.on_endpoint_status_changed(endpoint_id_2, IPV6, {"status": "up"}, async=True)
with patch("gevent.spawn_later", autospec=True) as m_spawn:
self.step_actor(self.rep)
self.rep._on_timer_pop(async=True)
self.step_actor(self.rep)
self.assertEqual(self.rep._older_dirty_endpoints, set())
self.assertEqual(self.rep._newer_dirty_endpoints, set())
self.rep.resync(async=True)
self.step_actor(self.rep)
self.assertEqual(self.rep._older_dirty_endpoints, set())
self.assertEqual(self.rep._newer_dirty_endpoints, set([endpoint_id, endpoint_id_2]))
def test_combine_statuses(self):
"""
Test the "truth table" for combining status reports.
"""
self.assert_combined_status(None, None, None)
self.assert_combined_status({"status": "up"}, None, {"status": "up"})
self.assert_combined_status({"status": "up"}, {"status": "up"},
{"status": "up"})
self.assert_combined_status({"status": "down"}, {"status": "up"},
{"status": "down"})
self.assert_combined_status({"status": "error"}, {"status": "up"},
{"status": "error"})
def assert_combined_status(self, a, b, expected):
# Should be symmetric so check the arguments both ways round.
for lhs, rhs in [(a, b), (b, a)]:
result = combine_statuses(lhs, rhs)
self.assertEqual(result, expected,
"Expected %r and %r to combine to %s but got %r" %
(lhs, rhs, expected, result))
