def test_delete_transient_ipt_error(self): """ Test that the dirty flag is left set if a delete fails. Future deletes that would normally be squashed trigger a retry. """ # First delete fails. self.m_ipt_updater.delete_chains.side_effect = \ FailedSystemCall("fail", ["foo"], 1, "", "") self.rules.on_profile_update(None, async=True) self.step_actor(self.rules) self._process_ipset_refs(set([])) self.m_ipt_updater.delete_chains.assert_called_once_with(set( RULES_1_CHAINS.keys()), async=False) # Failure should leave ProfileRules dirty. self.assertTrue(self.rules._dirty) # Update should trigger retry even though there was no change # of data. self.m_ipt_updater.reset_mock() self.m_ipt_updater.delete_chains.side_effect = None self.rules.on_profile_update(None, async=True) self.step_actor(self.rules) self._process_ipset_refs(set([])) self.m_ipt_updater.delete_chains.assert_called_once_with(set( RULES_1_CHAINS.keys()), async=False) # Successful delete leaves profile clean. self.assertFalse(self.rules._dirty)
def test_idempotent_update_transient_ipt_error(self): """ Test that the dirty flag is left set if the update fails. Future updates that would normally be squashed trigger a reprogram. """ # First update fails. self.m_ipt_updater.rewrite_chains.side_effect = \ FailedSystemCall("fail", ["foo"], 1, "", "") self.rules.on_profile_update(RULES_1, async=True) self.step_actor(self.rules) self._process_ipset_refs(set(["src-tag", "dst-tag"])) # Steps actor. self.m_ipt_updater.rewrite_chains.assert_called_once_with( RULES_1_CHAINS, {}, async=False) # Failure should leave ProfileRules dirty. self.assertTrue(self.rules._dirty) # Second update should trigger retry. self.m_ipt_updater.reset_mock() self.m_ipt_updater.rewrite_chains.side_effect = None self.rules.on_profile_update(RULES_1, async=True) self.step_actor(self.rules) self._process_ipset_refs(set([])) self.m_ipt_updater.rewrite_chains.assert_called_once_with( RULES_1_CHAINS, {}, async=False) # Success clears dirty flag. self.assertFalse(self.rules._dirty)
def test_insert_remove_tracking(self): fragment = "FOO --jump DROP" with patch.object(self.ipt, "_execute_iptables") as m_exec: m_exec.side_effect = [ # Insert. None, # Remove: requires an exception to terminate loop. None, FailedSystemCall("Message", [], 1, "", "line 2 failed"), # Insert. None, ] self.ipt.ensure_rule_inserted(fragment, async=True) self.step_actor(self.ipt) self.assertTrue(fragment in self.ipt._inserted_rule_fragments) self.assertTrue(fragment not in self.ipt._removed_rule_fragments) self.ipt.ensure_rule_removed(fragment, async=True) self.step_actor(self.ipt) self.assertTrue(fragment not in self.ipt._inserted_rule_fragments) self.assertTrue(fragment in self.ipt._removed_rule_fragments) self.ipt.ensure_rule_inserted(fragment, async=True) self.step_actor(self.ipt) self.assertTrue(fragment in self.ipt._inserted_rule_fragments) self.assertTrue(fragment not in self.ipt._removed_rule_fragments)
def test_load_nf_conntrack_fail(self): with patch("calico.felix.futils.check_call", autospec=True) as m_call: m_call.side_effect = FailedSystemCall(message="bad call", args=["conntrack", "-S"], retcode=1, stdout="", stderr="") frules.load_nf_conntrack() # Exception should be caught m_call.assert_called_once_with(["conntrack", "-S"])
def test_install_global_rules_retries_ipip(self): m_config = Mock() m_config.IFACE_PREFIX = ["tap"] m_config.IP_IN_IP_ENABLED = True with patch("calico.felix.frules._configure_ipip_device") as m_ipip: m_ipip.side_effect = FailedSystemCall("", [], 1, "", "") self.assertRaises(FailedSystemCall, frules.install_global_rules, m_config, None, None, 4) self.assertEqual(m_ipip.mock_calls, [call(m_config), call(m_config)])
def test_ensure_rule_removed_not_present(self): with patch.object(self.ipt, "_execute_iptables") as m_exec: m_exec.side_effect = iter([FailedSystemCall("Message", [], 1, "", "line 2 failed")]) self.ipt.ensure_rule_removed("FOO --jump DROP", async=True) self.step_actor(self.ipt) exp_call = call([ '*filter', '--delete FOO --jump DROP', 'COMMIT', ], fail_log_level=logging.DEBUG) self.assertEqual(m_exec.mock_calls, [exp_call])
def test_on_endpoint_update_delete_fail(self): combined_id = EndpointId("host_id", "orchestrator_id", "workload_id", "endpoint_id") ip_type = futils.IPV4 local_ep = self.get_local_endpoint(combined_id, ip_type) ips = ["1.2.3.4/32"] iface = "tapabcdef" data = { 'state': "active", 'endpoint': "endpoint_id", 'mac': stub_utils.get_mac(), 'name': iface, 'ipv4_nets': ips, 'profile_ids': ["prof1"] } # Report an initial update (endpoint creation) and check configured with mock.patch('calico.felix.devices.remove_conntrack_flows') as m_rem_conntrack,\ mock.patch('calico.felix.devices.set_routes') as m_set_routes,\ mock.patch('calico.felix.devices.configure_interface_ipv4') as m_conf,\ mock.patch('calico.felix.devices.interface_exists') as m_iface_exists,\ mock.patch('calico.felix.devices.interface_up') as m_iface_up: m_iface_exists.return_value = True m_iface_up.return_value = True local_ep.on_endpoint_update(data, async=True) self.step_actor(local_ep) self.assertEqual(local_ep._mac, data['mac']) m_conf.assert_called_once_with(iface) m_set_routes.assert_called_once_with(ip_type, set(["1.2.3.4"]), iface, data['mac'], reset_arp=True) self.assertFalse(m_rem_conntrack.called) # Send empty data, which deletes the endpoint. Raise an exception # from set_routes to check that it's handled. with mock.patch('calico.felix.devices.set_routes') as m_set_routes,\ mock.patch('calico.felix.devices.interface_exists', return_value=True),\ mock.patch('calico.felix.devices.remove_conntrack_flows') as m_rem_conntrack: m_set_routes.side_effect = FailedSystemCall("", [], 1, "", "") local_ep.on_endpoint_update(None, async=True) self.step_actor(local_ep) m_set_routes.assert_called_once_with(ip_type, set(), data["name"], None) # Should clean up conntrack entries for all IPs. m_rem_conntrack.assert_called_once_with( set(['1.2.3.4']), 4 )
def test_cleanup(self, m_check_call, m_list_ipsets): # We're testing the in-sync processing self.mgr.on_datamodel_in_sync(async=True) # Start with a couple ipsets. self.mgr.get_and_incref("foo", callback=self.on_ref_acquired, async=True) self.mgr.get_and_incref("bar", callback=self.on_ref_acquired, async=True) self.step_mgr() self.assertEqual(set(self.created_refs.keys()), set(["foo", "bar"])) # Notify ready so that the ipsets are marked as started. self._notify_ready(["foo", "bar"]) self.step_mgr() # Then decref "bar" so that it gets marked as stopping. self.mgr.decref("bar", async=True) self.step_mgr() self.assertEqual(self.mgr.stopping_objects_by_id, {"bar": set(self.created_refs["bar"])}) # Return mix of expected and unexpected ipsets. m_list_ipsets.return_value = [ "not-felix-foo", "felix-6-foo", "felix-6-bazzle", "felix-4-foo", "felix-4-bar", "felix-4-baz", "felix-4-biff", ] m_check_call.side_effect = iter([ # Exception on any individual call should be ignored. FailedSystemCall("Dummy", [], None, None, None), None, None, None, ]) self.mgr.cleanup(async=True) self.step_mgr() # Explicitly check that exactly the right delete calls were made. # assert_has_calls would ignore extra calls. self.assertEqual( sorted(m_check_call.mock_calls), sorted([ call(["ipset", "destroy", "felix-4-biff"]), call(["ipset", "destroy", "felix-4-baz"]), ]))
def test_replace_members_delete_fails(self, m_check_call): m_check_call.side_effect = iter( [FailedSystemCall("Blah", [], 1, None, "err"), None, None, None]) self.ipset.replace_members(set(["10.0.0.1"])) exp_calls = [ call(["ipset", "destroy", "foo-tmp"]), call(['ipset', 'list', 'foo-tmp']), call(['ipset', 'list', 'foo']), call(["ipset", "restore"], input_str='create foo-tmp hash:ip family inet ' 'maxelem 1048576 --exist\n' 'flush foo-tmp\n' 'add foo-tmp 10.0.0.1\n' 'swap foo foo-tmp\n' 'destroy foo-tmp\n' 'COMMIT\n') ] self.assertEqual(m_check_call.mock_calls, exp_calls)
def test_exception(self): """ Test exception when adding ipset value. """ description = "description" suffix = "suffix" rule_list = [{'cidr': "1.2.3.4/24"}] self.create_ipsets("inet") with mock.patch('calico.felix.test.stub_ipsets.add', side_effect=FailedSystemCall("oops", [], 1, "", "")): frules.update_ipsets(IPV4, description, suffix, rule_list, "ipset_addr", "ipset_port", "ipset_icmp", "tmp_ipset_addr", "tmp_ipset_port", "tmp_ipset_icmp") stub_ipsets.check_state(expected_ipsets)
def test_ensure_rule_inserted(self): fragment = "FOO --jump DROP" with patch.object(self.ipt, "_execute_iptables") as m_exec: m_exec.side_effect = iter([ FailedSystemCall("Message", [], 1, "", "line 2 failed"), None, None ]) self.ipt.ensure_rule_inserted(fragment, async=True) self.step_actor(self.ipt) self.assertEqual(m_exec.mock_calls, [ call([ "*filter", "--delete FOO --jump DROP", "--insert FOO --jump DROP", "COMMIT" ], fail_log_level=logging.DEBUG), call(["*filter", "--insert FOO --jump DROP", "COMMIT"]), ]) self.assertTrue(fragment in self.ipt._inserted_rule_fragments)
def test_delete_transient_ipt_error(self): """ Test that the dirty flag is left set if a delete fails. Future deletes that would normally be squashed trigger a retry. """ # First delete fails. self.rules.on_profile_update(RULES_1, async=True) self.step_actor(self.rules) self._process_ipset_refs(set(["dst-tag", "src-tag"])) self.m_ipt_updater.delete_chains.side_effect = \ FailedSystemCall("fail", ["foo"], 1, "", "") self.rules.on_profile_update(None, async=True) real_discard_all = self.rules._ipset_refs.discard_all with patch.object(self.rules._ipset_refs, "discard_all", wraps=real_discard_all) as m_discard: self.step_actor(self.rules) # Failure should prevent freeing of ipset refs. self.assertEqual(m_discard.mock_calls, []) self._process_ipset_refs(set([])) self.m_ipt_updater.delete_chains.assert_called_once_with(set( RULES_1_CHAINS.keys()), async=False) # Failure should leave ProfileRules dirty. self.assertTrue(self.rules._dirty) # Update should trigger retry even though there was no change # of data. self.m_ipt_updater.reset_mock() self.m_ipt_updater.delete_chains.side_effect = None self.rules.on_profile_update(None, async=True) with patch.object(self.rules._ipset_refs, "discard_all", wraps=real_discard_all) as m_discard: self.step_actor(self.rules) self.assertEqual(m_discard.mock_calls, [call()]) self._process_ipset_refs(set([])) self.m_ipt_updater.delete_chains.assert_called_once_with(set( RULES_1_CHAINS.keys()), async=False) # Successful delete leaves profile clean. self.assertFalse(self.rules._dirty)
def _handle_rule(self, rule): splits = rule.split(" ") ipt_op = splits[0] chain = splits[1] _log.debug("Rule op: %s, chain name: %s", ipt_op, chain) if ipt_op in ("--append", "-A", "--insert", "-I"): self.assert_chain_declared(chain, ipt_op) if ipt_op in ("--append", "-A"): self.new_contents[chain].append(rule) else: self.new_contents[chain].insert(0, rule) m = re.search(r'(?:--jump|-j|--goto|-g)\s+(\S+)', rule) if m: action = m.group(1) _log.debug("Action %s", action) if action not in STANDARD_ACTIONS: # Assume a dependent chain. self.new_dependencies[chain].add(action) elif ipt_op in ("--delete-chain", "-X"): self.assert_chain_declared(chain, ipt_op) del self.new_contents[chain] del self.new_dependencies[chain] elif ipt_op in ("--flush", "-F"): self.assert_chain_declared(chain, ipt_op) self.new_contents[chain] = [] self.new_dependencies[chain] = set() elif ipt_op in ("--delete", "-D"): self.assert_chain_declared(chain, ipt_op) for rule in self.new_contents.get(chain, []): rule_fragment = " ".join(splits[1:]) if rule.endswith(rule_fragment): self.new_contents[chain].remove(rule) break else: raise FailedSystemCall("Delete for non-existent rule", [], 1, "", "line 2 failed") else: raise AssertionError("Unknown operation %s; was expecting " "'--append|flush|delete|...' " % ipt_op)
class TestIpset(BaseTestCase): def setUp(self): super(TestIpset, self).setUp() self.ipset = Ipset("foo", "foo-tmp", "inet") @patch("calico.felix.futils.check_call", autospec=True) def test_replace_members(self, m_check_call): self.ipset.replace_members(set(["10.0.0.1"])) exp_calls = [ call(["ipset", "destroy", "foo-tmp"]), call(["ipset", "list", "foo"]), call(["ipset", "restore"], input_str='create foo-tmp hash:ip family inet ' 'maxelem 1048576 --exist\n' 'flush foo-tmp\n' 'add foo-tmp 10.0.0.1\n' 'swap foo foo-tmp\n' 'destroy foo-tmp\n' 'COMMIT\n') ] self.assertEqual(m_check_call.mock_calls, exp_calls) @patch("calico.felix.futils.check_call", autospec=True) def test_replace_members_delete_fails(self, m_check_call): m_check_call.side_effect = iter( [FailedSystemCall("Blah", [], 1, None, "err"), None, None, None]) self.ipset.replace_members(set(["10.0.0.1"])) exp_calls = [ call(["ipset", "destroy", "foo-tmp"]), call(['ipset', 'list', 'foo-tmp']), call(['ipset', 'list', 'foo']), call(["ipset", "restore"], input_str='create foo-tmp hash:ip family inet ' 'maxelem 1048576 --exist\n' 'flush foo-tmp\n' 'add foo-tmp 10.0.0.1\n' 'swap foo foo-tmp\n' 'destroy foo-tmp\n' 'COMMIT\n') ] self.assertEqual(m_check_call.mock_calls, exp_calls) @patch("calico.felix.futils.check_call", autospec=True) def test_apply_changes(self, m_check_call): added = set(["10.0.0.2"]) removed = set(["10.0.0.1"]) self.ipset.apply_changes(added, removed) self.assertEqual(m_check_call.mock_calls, [ call(["ipset", "restore"], input_str='del foo 10.0.0.1\n' 'add foo 10.0.0.2\n' 'COMMIT\n') ]) @patch("calico.felix.futils.check_call", autospec=True, side_effect=FailedSystemCall("Blah", [], None, None, "err")) def test_apply_changes_err(self, m_check_call): # First call to update_members will fail, leading to a retry. added = set(["10.0.0.2"]) removed = set(["10.0.0.1"]) self.assertRaises(FailedSystemCall, self.ipset.apply_changes, added, removed) @patch("calico.felix.futils.check_call", autospec=True) def test_ensure_exists(self, m_check_call): self.ipset.ensure_exists() m_check_call.assert_called_once_with( ["ipset", "restore"], input_str='create foo hash:ip family inet maxelem 1048576 --exist\n' 'COMMIT\n') @patch("calico.felix.futils.call_silent", autospec=True) def test_delete(self, m_call_silent): self.ipset.delete() self.assertEqual(m_call_silent.mock_calls, [ call(["ipset", "destroy", "foo"]), call(["ipset", "destroy", "foo-tmp"]), ]) @patch("calico.felix.futils.check_call", autospec=True) def test_list_ipset_names(self, m_check_call): m_check_call.return_value = CommandOutput(IPSET_LIST_OUTPUT, "") self.assertEqual(list_ipset_names(), ['felix-v4-calico_net', 'felix-v6-calico_net'])
def test_sync_to_ipset(self): members1 = ["1.2.3.4", "2.3.4.5"] members2 = ["1.2.3.6", "2.3.4.5"] # Each call to replace members causes a full update. _log.info("Calling replace_members() once...") self.actor.replace_members(members1, async=True) self.step_actor(self.actor) self.ipset.replace_members.assert_called_once_with(set(members1)) self.assertFalse(self.ipset.apply_changes.called) self.assertEqual(self.actor.members, set(members1)) self.assertFalse(self.actor._force_reprogram) self.ipset.reset_mock() _log.info("Calling replace_members() second time..") self.actor.replace_members(members2, async=True) self.step_actor(self.actor) self.ipset.replace_members.assert_called_once_with(set(members2)) self.assertFalse(self.ipset.apply_changes.called) self.assertEqual(self.actor.members, set(members2)) self.assertFalse(self.actor._force_reprogram) self.ipset.reset_mock() _log.info("Calling replace_members() third time with mixed " "add/remove..") self.actor.add_members(["5.6.7.8"], async=True) # Ignored self.actor.remove_members(["1.2.3.6"], async=True) self.actor.replace_members(members1, async=True) self.actor.add_members(["6.7.8.9"], async=True) # Should apply self.actor.remove_members(["2.3.4.5"], async=True) self.step_actor(self.actor) self.ipset.replace_members.assert_called_once_with( set([ "6.7.8.9", "1.2.3.4", ])) self.assertFalse(self.ipset.apply_changes.called) self.assertEqual(self.actor.members, set([ "6.7.8.9", "1.2.3.4", ])) self.assertFalse(self.actor._force_reprogram) self.ipset.reset_mock() _log.info("Calling add/remove.") self.actor.add_members(["5.6.7.8"], async=True) self.actor.remove_members(["1.2.3.4"], async=True) self.step_actor(self.actor) self.assertFalse(self.ipset.replace_members.called) self.assertEqual(self.ipset.apply_changes.mock_calls, [call(set(["5.6.7.8"]), set(["1.2.3.4"]))]) self.assertEqual(self.actor.members, set([ "6.7.8.9", "5.6.7.8", ])) self.assertFalse(self.actor._force_reprogram) self.ipset.reset_mock() _log.info("Calling add/remove with failure.") self.ipset.apply_changes.side_effect = FailedSystemCall( "", [], 1, "", "") self.actor.add_members(["6.7.8.10"], async=True) self.actor.remove_members(["5.6.7.8"], async=True) self.step_actor(self.actor) self.assertEqual(self.ipset.replace_members.mock_calls, [call(set(["6.7.8.9", "6.7.8.10"]))]) self.assertEqual(self.actor.members, set(["6.7.8.9", "6.7.8.10"])) self.assertFalse(self.actor._force_reprogram) self.ipset.reset_mock()
class TestIpset(BaseTestCase): def setUp(self): super(TestIpset, self).setUp() self.ipset = Ipset("foo", "foo-tmp", "inet") @patch("calico.felix.futils.check_call", autospec=True) def test_replace_members(self, m_check_call): self.ipset.replace_members(set(["10.0.0.1"])) m_check_call.assert_called_once_with( ["ipset", "restore"], input_str='create foo hash:ip family inet --exist\n' 'create foo-tmp hash:ip family inet --exist\n' 'flush foo-tmp\n' 'add foo-tmp 10.0.0.1\n' 'swap foo foo-tmp\n' 'destroy foo-tmp\n' 'COMMIT\n') @patch("calico.felix.futils.check_call", autospec=True) def test_update_members(self, m_check_call): old = set(["10.0.0.2"]) new = set(["10.0.0.1", "10.0.0.2"]) self.ipset.update_members(old, new) old = set(["10.0.0.1", "10.0.0.2"]) new = set(["10.0.0.1", "1.2.3.4"]) self.ipset.update_members(old, new) calls = [ call(["ipset", "restore"], input_str='add foo 10.0.0.1\nCOMMIT\n'), call(["ipset", "restore"], input_str='del foo 10.0.0.2\n' 'add foo 1.2.3.4\n' 'COMMIT\n') ] self.assertEqual(m_check_call.call_count, 2) m_check_call.assert_has_calls(calls) @patch("calico.felix.futils.check_call", autospec=True, side_effect=iter( [FailedSystemCall("Blah", [], None, None, "err"), None])) def test_update_members_err(self, m_check_call): # First call to update_members will fail, leading to a retry. old = set(["10.0.0.2"]) new = set(["10.0.0.1"]) self.ipset.update_members(old, new) calls = [ call(["ipset", "restore"], input_str='del foo 10.0.0.2\n' 'add foo 10.0.0.1\n' 'COMMIT\n'), call(["ipset", "restore"], input_str='create foo hash:ip family inet --exist\n' 'create foo-tmp hash:ip family inet --exist\n' 'flush foo-tmp\n' 'add foo-tmp 10.0.0.1\n' 'swap foo foo-tmp\n' 'destroy foo-tmp\n' 'COMMIT\n') ] self.assertEqual(m_check_call.call_count, 2) m_check_call.assert_has_calls(calls) @patch("calico.felix.futils.check_call", autospec=True) def test_ensure_exists(self, m_check_call): self.ipset.ensure_exists() m_check_call.assert_called_once_with( ["ipset", "restore"], input_str='create foo hash:ip family inet --exist\n' 'COMMIT\n') @patch("calico.felix.futils.call_silent", autospec=True) def test_delete(self, m_call_silent): self.ipset.delete() self.assertEqual(m_call_silent.mock_calls, [ call(["ipset", "destroy", "foo"]), call(["ipset", "destroy", "foo-tmp"]), ])
def test_on_interface_update_v6(self): combined_id = EndpointId("host_id", "orchestrator_id", "workload_id", "endpoint_id") ip_type = futils.IPV6 local_ep = self.get_local_endpoint(combined_id, ip_type) ips = ["1234::5678"] iface = "tapabcdef" data = { 'state': "active", 'endpoint': "endpoint_id", 'mac': stub_utils.get_mac(), 'name': iface, 'ipv6_nets': ips, 'profile_ids': ["prof1"] } # We can only get on_interface_update calls after the first # on_endpoint_update, so trigger that. with nested( mock.patch('calico.felix.devices.set_routes'), mock.patch('calico.felix.devices.configure_interface_ipv6'), mock.patch('calico.felix.devices.interface_up'), ) as [m_set_routes, m_conf, m_iface_up]: m_iface_up.return_value = False local_ep.on_endpoint_update(data, async=True) self.step_actor(local_ep) self.assertEqual(local_ep._mac, data['mac']) m_conf.assert_called_once_with(iface, None) m_set_routes.assert_called_once_with(ip_type, set(ips), iface, data['mac'], reset_arp=False) # Now pretend to get an interface update - does all the same work. with mock.patch('calico.felix.devices.set_routes') as m_set_routes: with mock.patch('calico.felix.devices.' 'configure_interface_ipv6') as m_conf: local_ep.on_interface_update(True, async=True) self.step_actor(local_ep) m_conf.assert_called_once_with(iface, None) m_set_routes.assert_called_once_with(ip_type, set(ips), iface, data['mac'], reset_arp=False) self.assertTrue(local_ep._device_in_sync) # Now cover the error cases... with mock.patch('calico.felix.devices.' 'configure_interface_ipv6') as m_conf: with mock.patch('calico.felix.devices.' 'interface_exists') as ifce_exists: with mock.patch('calico.felix.devices.' 'interface_up') as ifce_up: # Cycle through all the possibilities for the state. ifce_exists.side_effect = [True, False, True] ifce_up.side_effect = [True, False] m_conf.side_effect = FailedSystemCall("", [], 1, "", "") local_ep.on_interface_update(False, async=True) self.step_actor(local_ep) local_ep.on_interface_update(True, async=True) self.step_actor(local_ep) local_ep.on_interface_update(True, async=True) self.step_actor(local_ep) self.assertFalse(local_ep._device_in_sync)