def test_force_resync(self, m_spawn, m_etcd_watcher):
m_config = Mock(spec=Config)
m_hosts_ipset = Mock(spec=IpsetActor)
api = EtcdAPI(m_config, m_hosts_ipset)
api.force_resync(async=True)
self.step_actor(api)
self.assertTrue(m_etcd_watcher.return_value.resync_after_current_poll)
def test_periodic_resync_disabled(self, m_sleep, m_spawn, m_etcd_watcher):
m_etcd_watcher.return_value.configured = Mock(spec=Event)
m_config = Mock(spec=Config)
m_hosts_ipset = Mock(spec=IpsetActor)
api = EtcdAPI(m_config, m_hosts_ipset)
m_config.RESYNC_INTERVAL = 0
with patch.object(api, "force_resync") as m_force_resync:
m_force_resync.side_effect = Exception()
api._periodically_resync()
def test_periodic_resync_disabled(self, m_sleep, m_spawn, m_etcd_watcher):
m_etcd_watcher.return_value.configured = Mock(spec=Event)
m_config = Mock(spec=Config)
m_hosts_ipset = Mock(spec=IpsetActor)
api = EtcdAPI(m_config, m_hosts_ipset)
m_config.RESYNC_INTERVAL = 0
with patch.object(api, "force_resync") as m_force_resync:
m_force_resync.side_effect = Exception()
api._periodically_resync()
def test_force_resync(self, m_spawn, m_etcd_watcher):
m_config = Mock(spec=Config)
m_config.ETCD_ADDR = ETCD_ADDRESS
m_config.REPORT_ENDPOINT_STATUS = True
m_hosts_ipset = Mock(spec=IpsetActor)
api = EtcdAPI(m_config, m_hosts_ipset)
endpoint_id = EndpointId("foo", "bar", "baz", "biff")
with patch.object(api, "status_reporter") as m_status_rep:
api.force_resync(async=True)
self.step_actor(api)
m_status_rep.resync.assert_called_once_with(async=True)
self.assertTrue(m_etcd_watcher.return_value.resync_after_current_poll)
def test_force_resync(self, m_spawn, m_etcd_watcher):
m_config = Mock(spec=Config)
m_hosts_ipset = Mock(spec=IpsetActor)
api = EtcdAPI(m_config, m_hosts_ipset)
api.force_resync(async=True)
self.step_actor(api)
self.assertTrue(m_etcd_watcher.return_value.resync_after_current_poll)
def setUp(self):
super(TestEtcdReporting, self).setUp()
self.m_config = Mock()
self.m_config.IFACE_PREFIX = "tap"
self.m_config.ETCD_ADDR = "localhost:4001"
self.m_config.HOSTNAME = "hostname"
self.m_config.RESYNC_INTERVAL = 0
self.m_config.REPORTING_INTERVAL_SECS = 1
self.m_config.REPORTING_TTL_SECS = 10
self.m_hosts_ipset = Mock(spec=IpsetActor)
with patch("gevent.spawn", autospec=True):
with patch("calico.felix.fetcd._FelixEtcdWatcher", autospec=True):
with patch("calico.felix.fetcd.monotonic_time",
return_value=100):
self.api = EtcdAPI(self.m_config, self.m_hosts_ipset)
self.api._watcher.configured = Mock()
def setUp(self):
super(TestEtcdAPI, self).setUp()
self.m_config = Mock(spec=Config)
self.m_config.ETCD_ADDRS = [ETCD_ADDRESS]
self.m_config.ETCD_SCHEME = "http"
self.m_config.ETCD_KEY_FILE = None
self.m_config.ETCD_CERT_FILE = None
self.m_config.ETCD_CA_FILE = None
self.m_hosts_ipset = Mock(spec=IpsetActor)
with patch("calico.felix.fetcd._FelixEtcdWatcher",
autospec=True) as m_etcd_watcher:
with patch("gevent.spawn", autospec=True) as m_spawn:
self.api = EtcdAPI(self.m_config, self.m_hosts_ipset)
self.m_spawn = m_spawn
self.m_etcd_watcher = m_etcd_watcher.return_value
self.m_etcd_watcher.load_config = Mock(spec=Event)
self.m_etcd_watcher.begin_polling = Mock(spec=Event)
self.m_etcd_watcher.configured = Mock(spec=Event)
def test_force_resync(self, m_spawn, m_etcd_watcher):
m_config = Mock(spec=Config)
m_config.ETCD_ADDR = ETCD_ADDRESS
m_config.REPORT_ENDPOINT_STATUS = True
m_hosts_ipset = Mock(spec=IpsetActor)
api = EtcdAPI(m_config, m_hosts_ipset)
endpoint_id = EndpointId("foo", "bar", "baz", "biff")
with patch.object(api, "status_reporter") as m_status_rep:
api.force_resync(async=True)
self.step_actor(api)
m_status_rep.resync.assert_called_once_with(async=True)
self.assertTrue(m_etcd_watcher.return_value.resync_after_current_poll)
def test_create(self, m_spawn, m_etcd_watcher):
m_config = Mock(spec=Config)
m_hosts_ipset = Mock(spec=IpsetActor)
api = EtcdAPI(m_config, m_hosts_ipset)
m_etcd_watcher.assert_has_calls([
call(m_config, m_hosts_ipset).link(api._on_worker_died),
call(m_config, m_hosts_ipset).start(),
])
m_spawn.assert_has_calls([
call(api._periodically_resync),
call(api._periodically_resync).link_exception(api._on_worker_died)
])
def test_periodic_resync_mainline(self, m_sleep, m_spawn, m_etcd_watcher):
m_configured = Mock(spec=Event)
m_etcd_watcher.return_value.configured = m_configured
m_config = Mock(spec=Config)
m_hosts_ipset = Mock(spec=IpsetActor)
api = EtcdAPI(m_config, m_hosts_ipset)
m_config.RESYNC_INTERVAL = 10
with patch.object(api, "force_resync") as m_force_resync:
m_force_resync.side_effect = ExpectedException()
self.assertRaises(ExpectedException, api._periodically_resync)
m_configured.wait.assert_called_once_with()
m_sleep.assert_called_once_with(ANY)
sleep_time = m_sleep.call_args[0][0]
self.assertTrue(sleep_time >= 10)
self.assertTrue(sleep_time <= 12)
def setUp(self):
super(TestEtcdReporting, self).setUp()
self.m_config = Mock()
self.m_config.IFACE_PREFIX = "tap"
self.m_config.ETCD_ADDR = "localhost:4001"
self.m_config.HOSTNAME = "hostname"
self.m_config.RESYNC_INTERVAL = 0
self.m_config.REPORTING_INTERVAL_SECS = 1
self.m_config.REPORTING_TTL_SECS = 10
self.m_hosts_ipset = Mock(spec=IpsetActor)
with patch("gevent.spawn", autospec=True):
with patch("calico.felix.fetcd._FelixEtcdWatcher", autospec=True):
with patch("calico.felix.fetcd.monotonic_time",
return_value=100):
self.api = EtcdAPI(self.m_config, self.m_hosts_ipset)
self.api._watcher.configured = Mock()
def setUp(self):
super(TestEtcdAPI, self).setUp()
self.m_config = Mock(spec=Config)
self.m_config.ETCD_ADDRS = [ETCD_ADDRESS]
self.m_config.ETCD_SCHEME = "http"
self.m_config.ETCD_KEY_FILE = None
self.m_config.ETCD_CERT_FILE = None
self.m_config.ETCD_CA_FILE = None
self.m_hosts_ipset = Mock(spec=IpsetActor)
with patch("calico.felix.fetcd._FelixEtcdWatcher",
autospec=True) as m_etcd_watcher:
with patch("gevent.spawn", autospec=True) as m_spawn:
self.api = EtcdAPI(self.m_config, self.m_hosts_ipset)
self.m_spawn = m_spawn
self.m_etcd_watcher = m_etcd_watcher.return_value
self.m_etcd_watcher.load_config = Mock(spec=Event)
self.m_etcd_watcher.begin_polling = Mock(spec=Event)
self.m_etcd_watcher.configured = Mock(spec=Event)
def _main_greenlet(config):
"""
The root of our tree of greenlets. Responsible for restarting
its children if desired.
"""
try:
_log.info("Connecting to etcd to get our configuration.")
hosts_ipset_v4 = IpsetActor(HOSTS_IPSET_V4)
etcd_api = EtcdAPI(config, hosts_ipset_v4)
etcd_api.start()
# Ask the EtcdAPI to fill in the global config object before we
# proceed. We don't yet support config updates.
config_loaded = etcd_api.load_config(async=False)
config_loaded.wait()
# Ensure the Kernel's global options are correctly configured for
# Calico.
devices.configure_global_kernel_config()
_log.info("Main greenlet: Configuration loaded, starting remaining "
"actors...")
monitored_items = []
if config.PROM_METRICS_ENABLED:
httpd = HTTPServer(("0.0.0.0", config.PROM_METRICS_PORT),
MetricsHandler)
stats_server = gevent.Greenlet(httpd.serve_forever)
stats_server.start()
monitored_items.append(stats_server)
v4_filter_updater = IptablesUpdater("filter", ip_version=4,
config=config)
v4_nat_updater = IptablesUpdater("nat", ip_version=4, config=config)
v4_ipset_mgr = IpsetManager(IPV4, config)
v4_masq_manager = MasqueradeManager(IPV4, v4_nat_updater)
v4_rules_manager = RulesManager(config,
4,
v4_filter_updater,
v4_ipset_mgr)
v4_dispatch_chains = DispatchChains(config, 4, v4_filter_updater)
v4_fip_manager = FloatingIPManager(config, 4, v4_nat_updater)
v4_ep_manager = EndpointManager(config,
IPV4,
v4_filter_updater,
v4_dispatch_chains,
v4_rules_manager,
v4_fip_manager,
etcd_api.status_reporter)
cleanup_updaters = [v4_filter_updater, v4_nat_updater]
cleanup_ip_mgrs = [v4_ipset_mgr]
update_splitter_args = [v4_ipset_mgr,
v4_rules_manager,
v4_ep_manager,
v4_masq_manager,
v4_nat_updater]
v6_enabled = os.path.exists("/proc/sys/net/ipv6")
if v6_enabled:
v6_raw_updater = IptablesUpdater("raw", ip_version=6, config=config)
v6_filter_updater = IptablesUpdater("filter", ip_version=6,
config=config)
v6_nat_updater = IptablesUpdater("nat", ip_version=6, config=config)
v6_ipset_mgr = IpsetManager(IPV6, config)
v6_rules_manager = RulesManager(config,
6,
v6_filter_updater,
v6_ipset_mgr)
v6_dispatch_chains = DispatchChains(config, 6, v6_filter_updater)
v6_fip_manager = FloatingIPManager(config, 6, v6_nat_updater)
v6_ep_manager = EndpointManager(config,
IPV6,
v6_filter_updater,
v6_dispatch_chains,
v6_rules_manager,
v6_fip_manager,
etcd_api.status_reporter)
cleanup_updaters.append(v6_filter_updater)
cleanup_ip_mgrs.append(v6_ipset_mgr)
update_splitter_args += [v6_ipset_mgr,
v6_rules_manager,
v6_ep_manager,
v6_raw_updater,
v6_nat_updater]
cleanup_mgr = CleanupManager(config, cleanup_updaters, cleanup_ip_mgrs)
update_splitter_args.append(cleanup_mgr)
update_splitter = UpdateSplitter(update_splitter_args)
iface_watcher = InterfaceWatcher(update_splitter)
_log.info("Starting actors.")
hosts_ipset_v4.start()
cleanup_mgr.start()
v4_filter_updater.start()
v4_nat_updater.start()
v4_ipset_mgr.start()
v4_masq_manager.start()
v4_rules_manager.start()
v4_dispatch_chains.start()
v4_ep_manager.start()
v4_fip_manager.start()
if v6_enabled:
v6_raw_updater.start()
v6_filter_updater.start()
v6_ipset_mgr.start()
v6_nat_updater.start()
v6_rules_manager.start()
v6_dispatch_chains.start()
v6_ep_manager.start()
v6_fip_manager.start()
iface_watcher.start()
top_level_actors = [
hosts_ipset_v4,
cleanup_mgr,
v4_filter_updater,
v4_nat_updater,
v4_ipset_mgr,
v4_masq_manager,
v4_rules_manager,
v4_dispatch_chains,
v4_ep_manager,
v4_fip_manager,
iface_watcher,
etcd_api,
]
if v6_enabled:
top_level_actors += [
v6_raw_updater,
v6_filter_updater,
v6_nat_updater,
v6_ipset_mgr,
v6_rules_manager,
v6_dispatch_chains,
v6_ep_manager,
v6_fip_manager,
]
monitored_items += [actor.greenlet for actor in top_level_actors]
# Try to ensure that the nf_conntrack_netlink kernel module is present.
# This works around an issue[1] where the first call to the "conntrack"
# command fails while waiting for the module to load.
# [1] https://github.com/projectcalico/calico/issues/986
load_nf_conntrack()
# Install the global rules before we start polling for updates.
_log.info("Installing global rules.")
install_global_rules(config, v4_filter_updater, v4_nat_updater,
ip_version=4)
if v6_enabled:
install_global_rules(config, v6_filter_updater, v6_nat_updater,
ip_version=6, raw_updater=v6_raw_updater)
# Start polling for updates. These kicks make the actors poll
# indefinitely.
_log.info("Starting polling for interface and etcd updates.")
f = iface_watcher.watch_interfaces(async=True)
monitored_items.append(f)
etcd_api.start_watch(update_splitter, async=True)
# Register a SIG_USR handler to trigger a diags dump.
def dump_top_level_actors(log):
for a in top_level_actors:
# The output will include queue length and the like.
log.info("%s", a)
futils.register_diags("Top-level actors", dump_top_level_actors)
futils.register_process_statistics()
try:
gevent.signal(signal.SIGUSR1, functools.partial(futils.dump_diags))
except AttributeError:
# It doesn't matter too much if we fail to do this.
_log.warning("Unable to install diag dump handler")
pass
# Wait for something to fail.
_log.info("All top-level actors started, waiting on failures...")
stopped_greenlets_iter = gevent.iwait(monitored_items)
stopped_greenlet = next(stopped_greenlets_iter)
try:
stopped_greenlet.get()
except Exception:
_log.exception("Greenlet failed: %s", stopped_greenlet)
raise
else:
_log.error("Greenlet %s unexpectedly returned.", stopped_greenlet)
raise AssertionError("Greenlet unexpectedly returned")
except:
_log.exception("Exception killing main greenlet")
raise
class TestEtcdReporting(BaseTestCase):
def setUp(self):
super(TestEtcdReporting, self).setUp()
self.m_config = Mock()
self.m_config.IFACE_PREFIX = "tap"
self.m_config.ETCD_ADDR = "localhost:4001"
self.m_config.HOSTNAME = "hostname"
self.m_config.RESYNC_INTERVAL = 0
self.m_config.REPORTING_INTERVAL_SECS = 1
self.m_config.REPORTING_TTL_SECS = 10
self.m_hosts_ipset = Mock(spec=IpsetActor)
with patch("gevent.spawn", autospec=True):
with patch("calico.felix.fetcd._FelixEtcdWatcher", autospec=True):
with patch("calico.felix.fetcd.monotonic_time",
return_value=100):
self.api = EtcdAPI(self.m_config, self.m_hosts_ipset)
self.api._watcher.configured = Mock()
@patch("gevent.sleep", autospec=True)
def test_reporting_loop_mainline(self, m_sleep):
"""
Test the mainline function of the status reporting loop.
It should repeatedly call the _update_felix_status method,
retrying on various exceptions.
"""
with patch.object(self.api, "_update_felix_status") as m_update:
m_update.side_effect = [EtcdException, None, RuntimeError]
self.assertRaises(RuntimeError,
self.api._periodically_report_status)
self.assertEqual(m_update.mock_calls, [call(10)] * 3)
retry_call, jittered_call = m_sleep.mock_calls
self.assertEqual(retry_call, call(5))
_, (delay, ), _ = jittered_call
self.assertTrue(delay >= 1)
self.assertTrue(delay <= 1.1005)
def test_reporting_loop_disabled(self):
self.m_config.REPORTING_INTERVAL_SECS = 0
with patch.object(self.api, "_update_felix_status") as m_update:
m_update.side_effect = RuntimeError
self.api._periodically_report_status()
@patch("calico.felix.futils.datetime", autospec=True)
@patch("calico.felix.fetcd.monotonic_time", return_value=200)
def test_update_felix_status(self, m_monotime, m_datetime):
m_datetime.utcnow.return_value = datetime(2015, 9, 10, 2, 1, 53, 1234)
with patch.object(self.api.client, "set") as m_set:
self.api._update_felix_status(10)
self.api._update_felix_status(10)
# Should write two keys into etcd, one with a TTL and another with
# richer status.
self.assertEqual(m_set.mock_calls, [
call(
"/calico/felix/v1/host/hostname/last_reported_status",
JSONString({
"uptime": 100,
"time": "2015-09-10T02:01:53Z",
"first_update": True
})),
call("/calico/felix/v1/host/hostname/status",
JSONString({
"uptime": 100,
"time": "2015-09-10T02:01:53Z",
"first_update": True
}),
ttl=10),
call(
"/calico/felix/v1/host/hostname/last_reported_status",
JSONString({
"uptime": 100,
"time": "2015-09-10T02:01:53Z",
"first_update": False
})),
call("/calico/felix/v1/host/hostname/status",
JSONString({
"uptime": 100,
"time": "2015-09-10T02:01:53Z",
"first_update": False
}),
ttl=10),
])
class TestEtcdAPI(BaseTestCase):
def setUp(self):
super(TestEtcdAPI, self).setUp()
self.m_config = Mock(spec=Config)
self.m_config.ETCD_ADDRS = [ETCD_ADDRESS]
self.m_config.ETCD_SCHEME = "http"
self.m_config.ETCD_KEY_FILE = None
self.m_config.ETCD_CERT_FILE = None
self.m_config.ETCD_CA_FILE = None
self.m_hosts_ipset = Mock(spec=IpsetActor)
with patch("calico.felix.fetcd._FelixEtcdWatcher",
autospec=True) as m_etcd_watcher:
with patch("gevent.spawn", autospec=True) as m_spawn:
self.api = EtcdAPI(self.m_config, self.m_hosts_ipset)
self.m_spawn = m_spawn
self.m_etcd_watcher = m_etcd_watcher.return_value
self.m_etcd_watcher.load_config = Mock(spec=Event)
self.m_etcd_watcher.begin_polling = Mock(spec=Event)
self.m_etcd_watcher.configured = Mock(spec=Event)
def test_create(self):
self.m_etcd_watcher.assert_has_calls([
call.link(self.api._on_worker_died),
])
self.assertFalse(self.m_spawn.called)
def test_on_start(self):
with patch.object(self.api._resync_greenlet, "start") as m_resync_st, \
patch.object(self.api._status_reporting_greenlet, "start") as m_stat_start, \
patch.object(self.api.status_reporter, "start") as m_sr_start:
self.api._on_actor_started()
m_resync_st.assert_called_once_with()
m_stat_start.assert_called_once_with()
m_sr_start.assert_called_once_with()
self.m_etcd_watcher.start.assert_called_once_with()
@patch("gevent.sleep", autospec=True)
def test_periodic_resync_mainline(self, m_sleep):
self.m_config.RESYNC_INTERVAL = 10
m_configured = Mock(spec=Event)
self.m_etcd_watcher.configured = m_configured
with patch.object(self.api, "force_resync") as m_force_resync:
m_force_resync.side_effect = ExpectedException()
self.assertRaises(ExpectedException,
self.api._periodically_resync)
m_configured.wait.assert_called_once_with()
m_sleep.assert_called_once_with(ANY)
sleep_time = m_sleep.call_args[0][0]
self.assertTrue(sleep_time >= 10)
self.assertTrue(sleep_time <= 12)
@patch("gevent.sleep", autospec=True)
def test_periodic_resync_disabled(self, m_sleep):
self.m_config.RESYNC_INTERVAL = 0
self.m_etcd_watcher.configured = Mock(spec=Event)
with patch.object(self.api, "force_resync") as m_force_resync:
m_force_resync.side_effect = Exception()
self.api._periodically_resync()
def test_force_resync(self):
self.m_config.REPORT_ENDPOINT_STATUS = True
with patch.object(self.api, "status_reporter") as m_status_rep:
self.api.force_resync(async=True)
self.step_actor(self.api)
m_status_rep.resync.assert_called_once_with(async=True)
self.assertTrue(self.m_etcd_watcher.resync_requested)
def test_load_config(self):
result = self.api.load_config(async=True)
self.step_actor(self.api)
conf = result.get()
self.assertEqual(conf, self.m_etcd_watcher.configured)
self.m_etcd_watcher.load_config.set.assert_called_once_with()
def test_start_watch(self):
m_splitter = Mock()
self.api.load_config(async=True)
result = self.api.start_watch(m_splitter, async=True)
self.step_actor(self.api)
self.m_etcd_watcher.load_config.set.assert_called_once_with()
self.assertEqual(self.m_etcd_watcher.splitter, m_splitter)
self.m_etcd_watcher.begin_polling.set.assert_called_once_with()
@patch("sys.exit", autospec=True)
def test_on_worker_died(self, m_exit):
glet = gevent.spawn(lambda: None)
glet.link(self.api._on_worker_died)
glet.join(1)
m_exit.assert_called_once_with(1)
def _main_greenlet(config):
"""
The root of our tree of greenlets. Responsible for restarting
its children if desired.
"""
try:
_log.info("Connecting to etcd to get our configuration.")
hosts_ipset_v4 = IpsetActor(HOSTS_IPSET_V4)
etcd_api = EtcdAPI(config, hosts_ipset_v4)
etcd_api.start()
# Ask the EtcdAPI to fill in the global config object before we
# proceed. We don't yet support config updates.
config_loaded = etcd_api.load_config(async=False)
config_loaded.wait()
_log.info("Main greenlet: Configuration loaded, starting remaining "
"actors...")
v4_filter_updater = IptablesUpdater("filter", ip_version=4)
v4_nat_updater = IptablesUpdater("nat", ip_version=4)
v4_ipset_mgr = IpsetManager(IPV4)
v4_masq_manager = MasqueradeManager(IPV4, v4_nat_updater)
v4_rules_manager = RulesManager(4, v4_filter_updater, v4_ipset_mgr)
v4_dispatch_chains = DispatchChains(config, 4, v4_filter_updater)
v4_ep_manager = EndpointManager(config,
IPV4,
v4_filter_updater,
v4_dispatch_chains,
v4_rules_manager)
v6_filter_updater = IptablesUpdater("filter", ip_version=6)
v6_ipset_mgr = IpsetManager(IPV6)
v6_rules_manager = RulesManager(6, v6_filter_updater, v6_ipset_mgr)
v6_dispatch_chains = DispatchChains(config, 6, v6_filter_updater)
v6_ep_manager = EndpointManager(config,
IPV6,
v6_filter_updater,
v6_dispatch_chains,
v6_rules_manager)
update_splitter = UpdateSplitter(config,
[v4_ipset_mgr, v6_ipset_mgr],
[v4_rules_manager, v6_rules_manager],
[v4_ep_manager, v6_ep_manager],
[v4_filter_updater,
v6_filter_updater],
v4_masq_manager)
iface_watcher = InterfaceWatcher(update_splitter)
_log.info("Starting actors.")
hosts_ipset_v4.start()
update_splitter.start()
v4_filter_updater.start()
v4_nat_updater.start()
v4_ipset_mgr.start()
v4_masq_manager.start()
v4_rules_manager.start()
v4_dispatch_chains.start()
v4_ep_manager.start()
v6_filter_updater.start()
v6_ipset_mgr.start()
v6_rules_manager.start()
v6_dispatch_chains.start()
v6_ep_manager.start()
iface_watcher.start()
top_level_actors = [
hosts_ipset_v4,
update_splitter,
v4_nat_updater,
v4_filter_updater,
v4_nat_updater,
v4_ipset_mgr,
v4_masq_manager,
v4_rules_manager,
v4_dispatch_chains,
v4_ep_manager,
v6_filter_updater,
v6_ipset_mgr,
v6_rules_manager,
v6_dispatch_chains,
v6_ep_manager,
iface_watcher,
etcd_api,
]
monitored_items = [actor.greenlet for actor in top_level_actors]
# Install the global rules before we start polling for updates.
_log.info("Installing global rules.")
install_global_rules(config, v4_filter_updater, v6_filter_updater,
v4_nat_updater)
# Start polling for updates. These kicks make the actors poll
# indefinitely.
_log.info("Starting polling for interface and etcd updates.")
f = iface_watcher.watch_interfaces(async=True)
monitored_items.append(f)
etcd_api.start_watch(update_splitter, async=True)
# Register a SIG_USR handler to trigger a diags dump.
def dump_top_level_actors(log):
for a in top_level_actors:
# The output will include queue length and the like.
log.info("%s", a)
futils.register_diags("Top-level actors", dump_top_level_actors)
try:
gevent.signal(signal.SIGUSR1, functools.partial(futils.dump_diags))
except AttributeError:
# It doesn't matter too much if we fail to do this.
_log.warning("Unable to install diag dump handler")
pass
# Wait for something to fail.
_log.info("All top-level actors started, waiting on failures...")
stopped_greenlets_iter = gevent.iwait(monitored_items)
stopped_greenlet = next(stopped_greenlets_iter)
try:
stopped_greenlet.get()
except Exception:
_log.exception("Greenlet failed: %s", stopped_greenlet)
raise
else:
_log.error("Greenlet %s unexpectedly returned.", stopped_greenlet)
raise AssertionError("Greenlet unexpectedly returned")
except:
_log.exception("Exception killing main greenlet")
raise
class TestEtcdAPI(BaseTestCase):
def setUp(self):
super(TestEtcdAPI, self).setUp()
self.m_config = Mock(spec=Config)
self.m_config.ETCD_ADDRS = [ETCD_ADDRESS]
self.m_config.ETCD_SCHEME = "http"
self.m_config.ETCD_KEY_FILE = None
self.m_config.ETCD_CERT_FILE = None
self.m_config.ETCD_CA_FILE = None
self.m_hosts_ipset = Mock(spec=IpsetActor)
with patch("calico.felix.fetcd._FelixEtcdWatcher",
autospec=True) as m_etcd_watcher:
with patch("gevent.spawn", autospec=True) as m_spawn:
self.api = EtcdAPI(self.m_config, self.m_hosts_ipset)
self.m_spawn = m_spawn
self.m_etcd_watcher = m_etcd_watcher.return_value
self.m_etcd_watcher.load_config = Mock(spec=Event)
self.m_etcd_watcher.begin_polling = Mock(spec=Event)
self.m_etcd_watcher.configured = Mock(spec=Event)
def test_create(self):
self.m_etcd_watcher.assert_has_calls([
call.link(self.api._on_worker_died),
call.start(),
])
self.m_spawn.assert_has_calls([
call(self.api._periodically_resync),
call(self.api._periodically_resync).link_exception(
self.api._on_worker_died)
])
@patch("gevent.sleep", autospec=True)
def test_periodic_resync_mainline(self, m_sleep):
self.m_config.RESYNC_INTERVAL = 10
m_configured = Mock(spec=Event)
self.m_etcd_watcher.configured = m_configured
with patch.object(self.api, "force_resync") as m_force_resync:
m_force_resync.side_effect = ExpectedException()
self.assertRaises(ExpectedException, self.api._periodically_resync)
m_configured.wait.assert_called_once_with()
m_sleep.assert_called_once_with(ANY)
sleep_time = m_sleep.call_args[0][0]
self.assertTrue(sleep_time >= 10)
self.assertTrue(sleep_time <= 12)
@patch("gevent.sleep", autospec=True)
def test_periodic_resync_disabled(self, m_sleep):
self.m_config.RESYNC_INTERVAL = 0
self.m_etcd_watcher.configured = Mock(spec=Event)
with patch.object(self.api, "force_resync") as m_force_resync:
m_force_resync.side_effect = Exception()
self.api._periodically_resync()
def test_force_resync(self):
self.m_config.REPORT_ENDPOINT_STATUS = True
with patch.object(self.api, "status_reporter") as m_status_rep:
self.api.force_resync(async=True)
self.step_actor(self.api)
m_status_rep.resync.assert_called_once_with(async=True)
self.assertTrue(self.m_etcd_watcher.resync_requested)
def test_load_config(self):
result = self.api.load_config(async=True)
self.step_actor(self.api)
conf = result.get()
self.assertEqual(conf, self.m_etcd_watcher.configured)
self.m_etcd_watcher.load_config.set.assert_called_once_with()
def test_start_watch(self):
m_splitter = Mock()
self.api.load_config(async=True)
result = self.api.start_watch(m_splitter, async=True)
self.step_actor(self.api)
self.m_etcd_watcher.load_config.set.assert_called_once_with()
self.assertEqual(self.m_etcd_watcher.splitter, m_splitter)
self.m_etcd_watcher.begin_polling.set.assert_called_once_with()
@patch("sys.exit", autospec=True)
def test_on_worker_died(self, m_exit):
glet = gevent.spawn(lambda: None)
glet.link(self.api._on_worker_died)
glet.join(1)
m_exit.assert_called_once_with(1)
def _main_greenlet(config):
"""
The root of our tree of greenlets. Responsible for restarting
its children if desired.
"""
try:
_log.info("Connecting to etcd to get our configuration.")
hosts_ipset_v4 = IpsetActor(HOSTS_IPSET_V4)
etcd_api = EtcdAPI(config, hosts_ipset_v4)
etcd_api.start()
# Ask the EtcdAPI to fill in the global config object before we
# proceed. We don't yet support config updates.
config_loaded = etcd_api.load_config(async=False)
config_loaded.wait()
# Ensure the Kernel's global options are correctly configured for
# Calico.
devices.configure_global_kernel_config()
_log.info("Main greenlet: Configuration loaded, starting remaining "
"actors...")
v4_filter_updater = IptablesUpdater("filter", ip_version=4,
config=config)
v4_nat_updater = IptablesUpdater("nat", ip_version=4, config=config)
v4_ipset_mgr = IpsetManager(IPV4)
v4_masq_manager = MasqueradeManager(IPV4, v4_nat_updater)
v4_rules_manager = RulesManager(4, v4_filter_updater, v4_ipset_mgr)
v4_dispatch_chains = DispatchChains(config, 4, v4_filter_updater)
v4_ep_manager = EndpointManager(config,
IPV4,
v4_filter_updater,
v4_dispatch_chains,
v4_rules_manager,
etcd_api.status_reporter)
v6_raw_updater = IptablesUpdater("raw", ip_version=6, config=config)
v6_filter_updater = IptablesUpdater("filter", ip_version=6,
config=config)
v6_ipset_mgr = IpsetManager(IPV6)
v6_rules_manager = RulesManager(6, v6_filter_updater, v6_ipset_mgr)
v6_dispatch_chains = DispatchChains(config, 6, v6_filter_updater)
v6_ep_manager = EndpointManager(config,
IPV6,
v6_filter_updater,
v6_dispatch_chains,
v6_rules_manager,
etcd_api.status_reporter)
update_splitter = UpdateSplitter(config,
[v4_ipset_mgr, v6_ipset_mgr],
[v4_rules_manager, v6_rules_manager],
[v4_ep_manager, v6_ep_manager],
[v4_filter_updater,
v6_filter_updater,
v6_raw_updater,
v4_nat_updater],
v4_masq_manager)
iface_watcher = InterfaceWatcher(update_splitter)
_log.info("Starting actors.")
hosts_ipset_v4.start()
update_splitter.start()
v4_filter_updater.start()
v4_nat_updater.start()
v4_ipset_mgr.start()
v4_masq_manager.start()
v4_rules_manager.start()
v4_dispatch_chains.start()
v4_ep_manager.start()
v6_raw_updater.start()
v6_filter_updater.start()
v6_ipset_mgr.start()
v6_rules_manager.start()
v6_dispatch_chains.start()
v6_ep_manager.start()
iface_watcher.start()
top_level_actors = [
hosts_ipset_v4,
update_splitter,
v4_nat_updater,
v4_filter_updater,
v4_nat_updater,
v4_ipset_mgr,
v4_masq_manager,
v4_rules_manager,
v4_dispatch_chains,
v4_ep_manager,
v6_raw_updater,
v6_filter_updater,
v6_ipset_mgr,
v6_rules_manager,
v6_dispatch_chains,
v6_ep_manager,
iface_watcher,
etcd_api,
]
monitored_items = [actor.greenlet for actor in top_level_actors]
# Install the global rules before we start polling for updates.
_log.info("Installing global rules.")
install_global_rules(config, v4_filter_updater, v6_filter_updater,
v4_nat_updater, v6_raw_updater)
# Start polling for updates. These kicks make the actors poll
# indefinitely.
_log.info("Starting polling for interface and etcd updates.")
f = iface_watcher.watch_interfaces(async=True)
monitored_items.append(f)
etcd_api.start_watch(update_splitter, async=True)
# Register a SIG_USR handler to trigger a diags dump.
def dump_top_level_actors(log):
for a in top_level_actors:
# The output will include queue length and the like.
log.info("%s", a)
futils.register_diags("Top-level actors", dump_top_level_actors)
futils.register_process_statistics()
try:
gevent.signal(signal.SIGUSR1, functools.partial(futils.dump_diags))
except AttributeError:
# It doesn't matter too much if we fail to do this.
_log.warning("Unable to install diag dump handler")
pass
# Wait for something to fail.
_log.info("All top-level actors started, waiting on failures...")
stopped_greenlets_iter = gevent.iwait(monitored_items)
stopped_greenlet = next(stopped_greenlets_iter)
try:
stopped_greenlet.get()
except Exception:
_log.exception("Greenlet failed: %s", stopped_greenlet)
raise
else:
_log.error("Greenlet %s unexpectedly returned.", stopped_greenlet)
raise AssertionError("Greenlet unexpectedly returned")
except:
_log.exception("Exception killing main greenlet")
raise
def _main_greenlet(config):
"""
The root of our tree of greenlets. Responsible for restarting
its children if desired.
"""
try:
_log.info("Connecting to etcd to get our configuration.")
hosts_ipset_v4 = IpsetActor(HOSTS_IPSET_V4)
etcd_api = EtcdAPI(config, hosts_ipset_v4)
etcd_api.start()
# Ask the EtcdAPI to fill in the global config object before we
# proceed. We don't yet support config updates.
config_loaded = etcd_api.load_config(async=False)
config_loaded.wait()
# Ensure the Kernel's global options are correctly configured for
# Calico.
devices.configure_global_kernel_config()
_log.info("Main greenlet: Configuration loaded, starting remaining " "actors...")
monitored_items = []
if config.PROM_METRICS_ENABLED:
httpd = HTTPServer(("0.0.0.0", config.PROM_METRICS_PORT), MetricsHandler)
stats_server = gevent.Greenlet(httpd.serve_forever)
stats_server.start()
monitored_items.append(stats_server)
v4_filter_updater = IptablesUpdater("filter", ip_version=4, config=config)
v4_nat_updater = IptablesUpdater("nat", ip_version=4, config=config)
v4_ipset_mgr = IpsetManager(IPV4, config)
v4_masq_manager = MasqueradeManager(IPV4, v4_nat_updater)
v4_rules_manager = RulesManager(config, 4, v4_filter_updater, v4_ipset_mgr)
v4_dispatch_chains = DispatchChains(config, 4, v4_filter_updater)
v4_fip_manager = FloatingIPManager(config, 4, v4_nat_updater)
v4_ep_manager = EndpointManager(
config,
IPV4,
v4_filter_updater,
v4_dispatch_chains,
v4_rules_manager,
v4_fip_manager,
etcd_api.status_reporter,
)
cleanup_updaters = [v4_filter_updater, v4_nat_updater]
cleanup_ip_mgrs = [v4_ipset_mgr]
update_splitter_args = [v4_ipset_mgr, v4_rules_manager, v4_ep_manager, v4_masq_manager, v4_nat_updater]
v6_enabled = os.path.exists("/proc/sys/net/ipv6")
if v6_enabled:
v6_raw_updater = IptablesUpdater("raw", ip_version=6, config=config)
v6_filter_updater = IptablesUpdater("filter", ip_version=6, config=config)
v6_nat_updater = IptablesUpdater("nat", ip_version=6, config=config)
v6_ipset_mgr = IpsetManager(IPV6, config)
v6_rules_manager = RulesManager(config, 6, v6_filter_updater, v6_ipset_mgr)
v6_dispatch_chains = DispatchChains(config, 6, v6_filter_updater)
v6_fip_manager = FloatingIPManager(config, 6, v6_nat_updater)
v6_ep_manager = EndpointManager(
config,
IPV6,
v6_filter_updater,
v6_dispatch_chains,
v6_rules_manager,
v6_fip_manager,
etcd_api.status_reporter,
)
cleanup_updaters.append(v6_filter_updater)
cleanup_ip_mgrs.append(v6_ipset_mgr)
update_splitter_args += [v6_ipset_mgr, v6_rules_manager, v6_ep_manager, v6_raw_updater, v6_nat_updater]
cleanup_mgr = CleanupManager(config, cleanup_updaters, cleanup_ip_mgrs)
update_splitter_args.append(cleanup_mgr)
update_splitter = UpdateSplitter(update_splitter_args)
iface_watcher = InterfaceWatcher(update_splitter)
_log.info("Starting actors.")
hosts_ipset_v4.start()
cleanup_mgr.start()
v4_filter_updater.start()
v4_nat_updater.start()
v4_ipset_mgr.start()
v4_masq_manager.start()
v4_rules_manager.start()
v4_dispatch_chains.start()
v4_ep_manager.start()
v4_fip_manager.start()
if v6_enabled:
v6_raw_updater.start()
v6_filter_updater.start()
v6_ipset_mgr.start()
v6_nat_updater.start()
v6_rules_manager.start()
v6_dispatch_chains.start()
v6_ep_manager.start()
v6_fip_manager.start()
iface_watcher.start()
top_level_actors = [
hosts_ipset_v4,
cleanup_mgr,
v4_filter_updater,
v4_nat_updater,
v4_ipset_mgr,
v4_masq_manager,
v4_rules_manager,
v4_dispatch_chains,
v4_ep_manager,
v4_fip_manager,
iface_watcher,
etcd_api,
]
if v6_enabled:
top_level_actors += [
v6_raw_updater,
v6_filter_updater,
v6_nat_updater,
v6_ipset_mgr,
v6_rules_manager,
v6_dispatch_chains,
v6_ep_manager,
v6_fip_manager,
]
monitored_items += [actor.greenlet for actor in top_level_actors]
# Try to ensure that the nf_conntrack_netlink kernel module is present.
# This works around an issue[1] where the first call to the "conntrack"
# command fails while waiting for the module to load.
# [1] https://github.com/projectcalico/calico/issues/986
load_nf_conntrack()
# Install the global rules before we start polling for updates.
_log.info("Installing global rules.")
install_global_rules(config, v4_filter_updater, v4_nat_updater, ip_version=4)
if v6_enabled:
install_global_rules(config, v6_filter_updater, v6_nat_updater, ip_version=6, raw_updater=v6_raw_updater)
# Start polling for updates. These kicks make the actors poll
# indefinitely.
_log.info("Starting polling for interface and etcd updates.")
f = iface_watcher.watch_interfaces(async=True)
monitored_items.append(f)
etcd_api.start_watch(update_splitter, async=True)
# Register a SIG_USR handler to trigger a diags dump.
def dump_top_level_actors(log):
for a in top_level_actors:
# The output will include queue length and the like.
log.info("%s", a)
futils.register_diags("Top-level actors", dump_top_level_actors)
futils.register_process_statistics()
try:
gevent.signal(signal.SIGUSR1, functools.partial(futils.dump_diags))
except AttributeError:
# It doesn't matter too much if we fail to do this.
_log.warning("Unable to install diag dump handler")
pass
# Wait for something to fail.
_log.info("All top-level actors started, waiting on failures...")
stopped_greenlets_iter = gevent.iwait(monitored_items)
stopped_greenlet = next(stopped_greenlets_iter)
try:
stopped_greenlet.get()
except Exception:
_log.exception("Greenlet failed: %s", stopped_greenlet)
raise
else:
_log.error("Greenlet %s unexpectedly returned.", stopped_greenlet)
raise AssertionError("Greenlet unexpectedly returned")
except:
_log.exception("Exception killing main greenlet")
raise
class TestEtcdReporting(BaseTestCase):
def setUp(self):
super(TestEtcdReporting, self).setUp()
self.m_config = Mock()
self.m_config.IFACE_PREFIX = "tap"
self.m_config.ETCD_ADDR = "localhost:4001"
self.m_config.HOSTNAME = "hostname"
self.m_config.RESYNC_INTERVAL = 0
self.m_config.REPORTING_INTERVAL_SECS = 1
self.m_config.REPORTING_TTL_SECS = 10
self.m_hosts_ipset = Mock(spec=IpsetActor)
with patch("gevent.spawn", autospec=True):
with patch("calico.felix.fetcd._FelixEtcdWatcher", autospec=True):
with patch("calico.felix.fetcd.monotonic_time",
return_value=100):
self.api = EtcdAPI(self.m_config, self.m_hosts_ipset)
self.api._watcher.configured = Mock()
@patch("gevent.sleep", autospec=True)
def test_reporting_loop_mainline(self, m_sleep):
"""
Test the mainline function of the status reporting loop.
It should repeatedly call the _update_felix_status method,
retrying on various exceptions.
"""
with patch.object(self.api, "_update_felix_status") as m_update:
m_update.side_effect = [EtcdException, None, RuntimeError]
self.assertRaises(RuntimeError,
self.api._periodically_report_status)
self.assertEqual(m_update.mock_calls,
[call(10)] * 3)
retry_call, jittered_call = m_sleep.mock_calls
self.assertEqual(retry_call, call(5))
_, (delay,), _ = jittered_call
self.assertTrue(delay >= 1)
self.assertTrue(delay <= 1.1005)
def test_reporting_loop_disabled(self):
self.m_config.REPORTING_INTERVAL_SECS = 0
with patch.object(self.api, "_update_felix_status") as m_update:
m_update.side_effect = RuntimeError
self.api._periodically_report_status()
@patch("calico.felix.futils.datetime", autospec=True)
@patch("calico.felix.fetcd.monotonic_time", return_value=200)
def test_update_felix_status(self, m_monotime, m_datetime):
m_datetime.utcnow.return_value = datetime(2015, 9, 10, 2, 1, 53, 1234)
with patch.object(self.api.client, "set") as m_set:
self.api._update_felix_status(10)
self.api._update_felix_status(10)
# Should write two keys into etcd, one with a TTL and another with
# richer status.
self.assertEqual(m_set.mock_calls, [
call("/calico/felix/v1/host/hostname/last_reported_status",
JSONString({"uptime": 100,
"time": "2015-09-10T02:01:53Z",
"first_update": True})),
call("/calico/felix/v1/host/hostname/status",
JSONString({"uptime": 100,
"time": "2015-09-10T02:01:53Z",
"first_update": True}), ttl=10),
call("/calico/felix/v1/host/hostname/last_reported_status",
JSONString({"uptime": 100,
"time": "2015-09-10T02:01:53Z",
"first_update": False})),
call("/calico/felix/v1/host/hostname/status",
JSONString({"uptime": 100,
"time": "2015-09-10T02:01:53Z",
"first_update": False}), ttl=10),
])