class TestEtcdStatusReporter(BaseTestCase): def setUp(self): super(TestEtcdStatusReporter, self).setUp() self.m_config = Mock(spec=Config) self.m_config.ETCD_ADDRS = [ETCD_ADDRESS] self.m_config.ETCD_SCHEME = "http" self.m_config.ETCD_KEY_FILE = None self.m_config.ETCD_CERT_FILE = None self.m_config.ETCD_CA_FILE = None self.m_config.HOSTNAME = "foo" self.m_config.REPORT_ENDPOINT_STATUS = True self.m_config.ENDPOINT_REPORT_DELAY = 1 self.m_client = Mock() self.rep = EtcdStatusReporter(self.m_config) self.rep.client = self.m_client def test_on_endpoint_status_mainline(self): # Send in an endpoint status update. endpoint_id = WloadEndpointId("foo", "bar", "baz", "biff") with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True) self.step_actor(self.rep) # Should record the status. self.assertEqual( self.rep._endpoint_status[IPV4], { endpoint_id: {"status": "up"} } ) # And do a write. self.assertEqual( self.m_client.set.mock_calls, [call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff", JSONString({"status": "up"}))] ) # Since we did a write, the rate limit timer should be scheduled. self.assertEqual( m_spawn.mock_calls, [call(ANY, self.rep._on_timer_pop, async=True)] ) self.assertTrue(self.rep._timer_scheduled) self.assertFalse(self.rep._reporting_allowed) # Send in another update, shouldn't get written until we pop the timer. self.m_client.reset_mock() with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep.on_endpoint_status_changed(endpoint_id, IPV4, None, async=True) self.step_actor(self.rep) self.assertFalse(self.m_client.set.called) # Timer already scheduled, shouldn't get rescheduled. self.assertFalse(m_spawn.called) # Pop the timer, should trigger write and reschedule. with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep._on_timer_pop(async=True) self.step_actor(self.rep) self.maxDiff = 10000 self.assertEqual( self.m_client.delete.mock_calls, [ call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/" "biff"), call("calico/felix/v1/host/foo/workload/bar/baz/endpoint", dir=True, timeout=5), call("calico/felix/v1/host/foo/workload/bar/baz", dir=True, timeout=5), call("calico/felix/v1/host/foo/workload/bar", dir=True, timeout=5), call("calico/felix/v1/host/foo/workload", dir=True, timeout=5), ] ) # Rate limit timer should be scheduled. self.assertEqual( m_spawn.mock_calls, [call(ANY, self.rep._on_timer_pop, async=True)] ) spawn_delay = m_spawn.call_args[0][0] self.assertTrue(spawn_delay >= 0.89999) self.assertTrue(spawn_delay <= 1.10001) self.assertTrue(self.rep._timer_scheduled) self.assertFalse(self.rep._reporting_allowed) # Cache should be cleaned up. self.assertEqual(self.rep._endpoint_status[IPV4], {}) # Nothing queued. self.assertEqual(self.rep._newer_dirty_endpoints, set()) self.assertEqual(self.rep._older_dirty_endpoints, set()) def test_mark_endpoint_dirty_already_dirty(self): endpoint_id = WloadEndpointId("a", "b", "c", "d") self.rep._older_dirty_endpoints.add(endpoint_id) self.rep._mark_endpoint_dirty(endpoint_id) self.assertFalse(endpoint_id in self.rep._newer_dirty_endpoints) def test_on_endpoint_status_failure(self): # Send in an endpoint status update. endpoint_id = WloadEndpointId("foo", "bar", "baz", "biff") self.m_client.set.side_effect = EtcdException() with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True) self.step_actor(self.rep) # Should do the write. self.assertEqual( self.m_client.set.mock_calls, [call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff", JSONString({"status": "up"}))] ) # But endpoint should be re-queued in the newer set. self.assertEqual(self.rep._newer_dirty_endpoints, set([endpoint_id])) self.assertEqual(self.rep._older_dirty_endpoints, set()) def test_on_endpoint_status_changed_disabled(self): self.m_config.REPORT_ENDPOINT_STATUS = False endpoint_id = WloadEndpointId("foo", "bar", "baz", "biff") with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True) self.step_actor(self.rep) self.assertFalse(m_spawn.called) self.assertEqual(self.rep._endpoint_status[IPV4], {}) # Nothing queued. self.assertEqual(self.rep._newer_dirty_endpoints, set()) self.assertEqual(self.rep._older_dirty_endpoints, set()) def test_on_endpoint_status_v4_v6(self): # Send in endpoint status updates for v4 and v6. endpoint_id = WloadEndpointId("foo", "bar", "baz", "biff") with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True) self.rep.on_endpoint_status_changed(endpoint_id, IPV6, {"status": "down"}, async=True) self.step_actor(self.rep) # Should record the status. self.assertEqual( self.rep._endpoint_status, { IPV4: {endpoint_id: {"status": "up"}}, IPV6: {endpoint_id: {"status": "down"}}, } ) # And do a write. self.assertEqual( self.m_client.set.mock_calls, [call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff", JSONString({"status": "down"}))] ) def test_resync(self): endpoint_id = WloadEndpointId("foo", "bar", "baz", "biff") self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True) endpoint_id_2 = WloadEndpointId("foo", "bar", "baz", "boff") self.rep.on_endpoint_status_changed(endpoint_id_2, IPV6, {"status": "up"}, async=True) with patch("gevent.spawn_later", autospec=True) as m_spawn: self.step_actor(self.rep) self.rep._on_timer_pop(async=True) self.step_actor(self.rep) self.assertEqual(self.rep._older_dirty_endpoints, set()) self.assertEqual(self.rep._newer_dirty_endpoints, set()) self.rep.resync(async=True) self.step_actor(self.rep) self.assertEqual(self.rep._older_dirty_endpoints, set()) self.assertEqual(self.rep._newer_dirty_endpoints, set([endpoint_id, endpoint_id_2])) def test_combine_statuses(self): """ Test the "truth table" for combining status reports. """ self.assert_combined_status(None, None, None) self.assert_combined_status({"status": "up"}, None, {"status": "up"}) self.assert_combined_status({"status": "up"}, {"status": "up"}, {"status": "up"}) self.assert_combined_status({"status": "down"}, {"status": "up"}, {"status": "down"}) self.assert_combined_status({"status": "error"}, {"status": "up"}, {"status": "error"}) def assert_combined_status(self, a, b, expected): # Should be symmetric so check the arguments both ways round. for lhs, rhs in [(a, b), (b, a)]: result = combine_statuses(lhs, rhs) self.assertEqual(result, expected, "Expected %r and %r to combine to %s but got %r" % (lhs, rhs, expected, result)) def test_clean_up_endpoint_status(self): self.m_config.REPORT_ENDPOINT_STATUS = True ep_id = WloadEndpointId("foo", "openstack", "workloadid", "endpointid") empty_dir = Mock() empty_dir.key = ("/calico/felix/v1/host/foo/workload/" "openstack/foobar") empty_dir.dir = True missing_ep = Mock() missing_ep.key = ("/calico/felix/v1/host/foo/workload/" "openstack/aworkload/endpoint/anendpoint") self.m_client.read.return_value.leaves = [ empty_dir, missing_ep, ] with patch.object(self.rep, "_mark_endpoint_dirty") as m_mark: self.rep.clean_up_endpoint_statuses(async=True) self.step_actor(self.rep) # Missing endpoint should have been marked for cleanup. m_mark.assert_called_once_with( WloadEndpointId("foo", "openstack", "aworkload", "anendpoint") ) def test_clean_up_endpoint_status_etcd_error(self): self.m_config.REPORT_ENDPOINT_STATUS = True with patch.object(self.rep, "_attempt_cleanup") as m_clean: m_clean.side_effect = EtcdException() self.rep.clean_up_endpoint_statuses(async=True) self.step_actor(self.rep) self.assertTrue(self.rep._cleanup_pending) def test_clean_up_endpoint_status_not_found(self): self.m_config.REPORT_ENDPOINT_STATUS = True self.m_client.read.side_effect = etcd.EtcdKeyNotFound() with patch.object(self.rep, "_mark_endpoint_dirty") as m_mark: self.rep.clean_up_endpoint_statuses(async=True) self.step_actor(self.rep) self.assertFalse(m_mark.called) def test_clean_up_endpoint_status_disabled(self): self.m_config.REPORT_ENDPOINT_STATUS = False self.m_client.read.side_effect = self.failureException self.rep.clean_up_endpoint_statuses(async=True) self.step_actor(self.rep)
class TestEtcdStatusReporter(BaseTestCase): def setUp(self): super(TestEtcdStatusReporter, self).setUp() self.m_config = Mock(spec=Config) self.m_config.ETCD_ADDR = ETCD_ADDRESS self.m_config.HOSTNAME = "foo" self.m_config.REPORT_ENDPOINT_STATUS = True self.m_config.ENDPOINT_REPORT_DELAY = 1 self.m_client = Mock() self.rep = EtcdStatusReporter(self.m_config) self.rep.client = self.m_client def test_on_endpoint_status_mainline(self): # Send in an endpoint status update. endpoint_id = EndpointId("foo", "bar", "baz", "biff") with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True) self.step_actor(self.rep) # Should record the status. self.assertEqual( self.rep._endpoint_status[IPV4], { endpoint_id: {"status": "up"} } ) # And do a write. self.assertEqual( self.m_client.set.mock_calls, [call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff", JSONString({"status": "up"}))] ) # Since we did a write, the rate limit timer should be scheduled. self.assertEqual( m_spawn.mock_calls, [call(ANY, self.rep._on_timer_pop, async=True)] ) self.assertTrue(self.rep._timer_scheduled) self.assertFalse(self.rep._reporting_allowed) # Send in another update, shouldn't get written until we pop the timer. self.m_client.reset_mock() with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep.on_endpoint_status_changed(endpoint_id, IPV4, None, async=True) self.step_actor(self.rep) self.assertFalse(self.m_client.set.called) # Timer already scheduled, shouldn't get rescheduled. self.assertFalse(m_spawn.called) # Pop the timer, should trigger write and reschedule. with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep._on_timer_pop(async=True) self.step_actor(self.rep) self.maxDiff = 10000 self.assertEqual( self.m_client.delete.mock_calls, [ call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/" "biff"), call("calico/felix/v1/host/foo/workload/bar/baz/endpoint", dir=True, timeout=5), call("calico/felix/v1/host/foo/workload/bar/baz", dir=True, timeout=5), call("calico/felix/v1/host/foo/workload/bar", dir=True, timeout=5), call("calico/felix/v1/host/foo/workload", dir=True, timeout=5), ] ) # Rate limit timer should be scheduled. self.assertEqual( m_spawn.mock_calls, [call(ANY, self.rep._on_timer_pop, async=True)] ) spawn_delay = m_spawn.call_args[0][0] self.assertTrue(spawn_delay >= 0.89999) self.assertTrue(spawn_delay <= 1.10001) self.assertTrue(self.rep._timer_scheduled) self.assertFalse(self.rep._reporting_allowed) # Cache should be cleaned up. self.assertEqual(self.rep._endpoint_status[IPV4], {}) # Nothing queued. self.assertEqual(self.rep._newer_dirty_endpoints, set()) self.assertEqual(self.rep._older_dirty_endpoints, set()) def test_on_endpoint_status_failure(self): # Send in an endpoint status update. endpoint_id = EndpointId("foo", "bar", "baz", "biff") self.m_client.set.side_effect = EtcdException() with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True) self.step_actor(self.rep) # Should do the write. self.assertEqual( self.m_client.set.mock_calls, [call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff", JSONString({"status": "up"}))] ) # But endpoint should be re-queued in the newer set. self.assertEqual(self.rep._newer_dirty_endpoints, set([endpoint_id])) self.assertEqual(self.rep._older_dirty_endpoints, set()) def test_on_endpoint_status_changed_disabled(self): self.m_config.REPORT_ENDPOINT_STATUS = False endpoint_id = EndpointId("foo", "bar", "baz", "biff") with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True) self.step_actor(self.rep) self.assertFalse(m_spawn.called) self.assertEqual(self.rep._endpoint_status[IPV4], {}) # Nothing queued. self.assertEqual(self.rep._newer_dirty_endpoints, set()) self.assertEqual(self.rep._older_dirty_endpoints, set()) def test_on_endpoint_status_v4_v6(self): # Send in endpoint status updates for v4 and v6. endpoint_id = EndpointId("foo", "bar", "baz", "biff") with patch("gevent.spawn_later", autospec=True) as m_spawn: self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True) self.rep.on_endpoint_status_changed(endpoint_id, IPV6, {"status": "down"}, async=True) self.step_actor(self.rep) # Should record the status. self.assertEqual( self.rep._endpoint_status, { IPV4: {endpoint_id: {"status": "up"}}, IPV6: {endpoint_id: {"status": "down"}}, } ) # And do a write. self.assertEqual( self.m_client.set.mock_calls, [call("/calico/felix/v1/host/foo/workload/bar/baz/endpoint/biff", JSONString({"status": "down"}))] ) def test_resync(self): endpoint_id = EndpointId("foo", "bar", "baz", "biff") self.rep.on_endpoint_status_changed(endpoint_id, IPV4, {"status": "up"}, async=True) endpoint_id_2 = EndpointId("foo", "bar", "baz", "boff") self.rep.on_endpoint_status_changed(endpoint_id_2, IPV6, {"status": "up"}, async=True) with patch("gevent.spawn_later", autospec=True) as m_spawn: self.step_actor(self.rep) self.rep._on_timer_pop(async=True) self.step_actor(self.rep) self.assertEqual(self.rep._older_dirty_endpoints, set()) self.assertEqual(self.rep._newer_dirty_endpoints, set()) self.rep.resync(async=True) self.step_actor(self.rep) self.assertEqual(self.rep._older_dirty_endpoints, set()) self.assertEqual(self.rep._newer_dirty_endpoints, set([endpoint_id, endpoint_id_2])) def test_combine_statuses(self): """ Test the "truth table" for combining status reports. """ self.assert_combined_status(None, None, None) self.assert_combined_status({"status": "up"}, None, {"status": "up"}) self.assert_combined_status({"status": "up"}, {"status": "up"}, {"status": "up"}) self.assert_combined_status({"status": "down"}, {"status": "up"}, {"status": "down"}) self.assert_combined_status({"status": "error"}, {"status": "up"}, {"status": "error"}) def assert_combined_status(self, a, b, expected): # Should be symmetric so check the arguments both ways round. for lhs, rhs in [(a, b), (b, a)]: result = combine_statuses(lhs, rhs) self.assertEqual(result, expected, "Expected %r and %r to combine to %s but got %r" % (lhs, rhs, expected, result))