def test_delete_transient_ipt_error(self):
"""
Test that the dirty flag is left set if a delete fails. Future
deletes that would normally be squashed trigger a retry.
"""
# First delete fails.
self.m_ipt_updater.delete_chains.side_effect = \
FailedSystemCall("fail", ["foo"], 1, "", "")
self.rules.on_profile_update(None, async=True)
self.step_actor(self.rules)
self._process_ipset_refs(set([]))
self.m_ipt_updater.delete_chains.assert_called_once_with(set(
RULES_1_CHAINS.keys()),
async=False)
# Failure should leave ProfileRules dirty.
self.assertTrue(self.rules._dirty)
# Update should trigger retry even though there was no change
# of data.
self.m_ipt_updater.reset_mock()
self.m_ipt_updater.delete_chains.side_effect = None
self.rules.on_profile_update(None, async=True)
self.step_actor(self.rules)
self._process_ipset_refs(set([]))
self.m_ipt_updater.delete_chains.assert_called_once_with(set(
RULES_1_CHAINS.keys()),
async=False)
# Successful delete leaves profile clean.
self.assertFalse(self.rules._dirty)
def test_idempotent_update_transient_ipt_error(self):
"""
Test that the dirty flag is left set if the update fails. Future
updates that would normally be squashed trigger a reprogram.
"""
# First update fails.
self.m_ipt_updater.rewrite_chains.side_effect = \
FailedSystemCall("fail", ["foo"], 1, "", "")
self.rules.on_profile_update(RULES_1, async=True)
self.step_actor(self.rules)
self._process_ipset_refs(set(["src-tag", "dst-tag"])) # Steps actor.
self.m_ipt_updater.rewrite_chains.assert_called_once_with(
RULES_1_CHAINS, {}, async=False)
# Failure should leave ProfileRules dirty.
self.assertTrue(self.rules._dirty)
# Second update should trigger retry.
self.m_ipt_updater.reset_mock()
self.m_ipt_updater.rewrite_chains.side_effect = None
self.rules.on_profile_update(RULES_1, async=True)
self.step_actor(self.rules)
self._process_ipset_refs(set([]))
self.m_ipt_updater.rewrite_chains.assert_called_once_with(
RULES_1_CHAINS, {}, async=False)
# Success clears dirty flag.
self.assertFalse(self.rules._dirty)
def test_insert_remove_tracking(self):
fragment = "FOO --jump DROP"
with patch.object(self.ipt, "_execute_iptables") as m_exec:
m_exec.side_effect = [
# Insert.
None,
# Remove: requires an exception to terminate loop.
None,
FailedSystemCall("Message", [], 1, "", "line 2 failed"),
# Insert.
None,
]
self.ipt.ensure_rule_inserted(fragment, async=True)
self.step_actor(self.ipt)
self.assertTrue(fragment in self.ipt._inserted_rule_fragments)
self.assertTrue(fragment not in self.ipt._removed_rule_fragments)
self.ipt.ensure_rule_removed(fragment, async=True)
self.step_actor(self.ipt)
self.assertTrue(fragment not in self.ipt._inserted_rule_fragments)
self.assertTrue(fragment in self.ipt._removed_rule_fragments)
self.ipt.ensure_rule_inserted(fragment, async=True)
self.step_actor(self.ipt)
self.assertTrue(fragment in self.ipt._inserted_rule_fragments)
self.assertTrue(fragment not in self.ipt._removed_rule_fragments)
def test_load_nf_conntrack_fail(self):
with patch("calico.felix.futils.check_call", autospec=True) as m_call:
m_call.side_effect = FailedSystemCall(message="bad call",
args=["conntrack", "-S"],
retcode=1,
stdout="", stderr="")
frules.load_nf_conntrack() # Exception should be caught
m_call.assert_called_once_with(["conntrack", "-S"])
def test_install_global_rules_retries_ipip(self):
m_config = Mock()
m_config.IFACE_PREFIX = ["tap"]
m_config.IP_IN_IP_ENABLED = True
with patch("calico.felix.frules._configure_ipip_device") as m_ipip:
m_ipip.side_effect = FailedSystemCall("", [], 1, "", "")
self.assertRaises(FailedSystemCall, frules.install_global_rules,
m_config, None, None, 4)
self.assertEqual(m_ipip.mock_calls,
[call(m_config), call(m_config)])
def test_ensure_rule_removed_not_present(self):
with patch.object(self.ipt, "_execute_iptables") as m_exec:
m_exec.side_effect = iter([FailedSystemCall("Message", [], 1, "",
"line 2 failed")])
self.ipt.ensure_rule_removed("FOO --jump DROP", async=True)
self.step_actor(self.ipt)
exp_call = call([
'*filter',
'--delete FOO --jump DROP',
'COMMIT',
], fail_log_level=logging.DEBUG)
self.assertEqual(m_exec.mock_calls, [exp_call])
def test_on_endpoint_update_delete_fail(self):
combined_id = EndpointId("host_id", "orchestrator_id",
"workload_id", "endpoint_id")
ip_type = futils.IPV4
local_ep = self.get_local_endpoint(combined_id, ip_type)
ips = ["1.2.3.4/32"]
iface = "tapabcdef"
data = {
'state': "active",
'endpoint': "endpoint_id",
'mac': stub_utils.get_mac(),
'name': iface,
'ipv4_nets': ips,
'profile_ids': ["prof1"]
}
# Report an initial update (endpoint creation) and check configured
with mock.patch('calico.felix.devices.remove_conntrack_flows') as m_rem_conntrack,\
mock.patch('calico.felix.devices.set_routes') as m_set_routes,\
mock.patch('calico.felix.devices.configure_interface_ipv4') as m_conf,\
mock.patch('calico.felix.devices.interface_exists') as m_iface_exists,\
mock.patch('calico.felix.devices.interface_up') as m_iface_up:
m_iface_exists.return_value = True
m_iface_up.return_value = True
local_ep.on_endpoint_update(data, async=True)
self.step_actor(local_ep)
self.assertEqual(local_ep._mac, data['mac'])
m_conf.assert_called_once_with(iface)
m_set_routes.assert_called_once_with(ip_type,
set(["1.2.3.4"]),
iface,
data['mac'],
reset_arp=True)
self.assertFalse(m_rem_conntrack.called)
# Send empty data, which deletes the endpoint. Raise an exception
# from set_routes to check that it's handled.
with mock.patch('calico.felix.devices.set_routes') as m_set_routes,\
mock.patch('calico.felix.devices.interface_exists', return_value=True),\
mock.patch('calico.felix.devices.remove_conntrack_flows') as m_rem_conntrack:
m_set_routes.side_effect = FailedSystemCall("", [], 1, "", "")
local_ep.on_endpoint_update(None, async=True)
self.step_actor(local_ep)
m_set_routes.assert_called_once_with(ip_type, set(),
data["name"], None)
# Should clean up conntrack entries for all IPs.
m_rem_conntrack.assert_called_once_with(
set(['1.2.3.4']), 4
)
def test_cleanup(self, m_check_call, m_list_ipsets):
# We're testing the in-sync processing
self.mgr.on_datamodel_in_sync(async=True)
# Start with a couple ipsets.
self.mgr.get_and_incref("foo",
callback=self.on_ref_acquired,
async=True)
self.mgr.get_and_incref("bar",
callback=self.on_ref_acquired,
async=True)
self.step_mgr()
self.assertEqual(set(self.created_refs.keys()), set(["foo", "bar"]))
# Notify ready so that the ipsets are marked as started.
self._notify_ready(["foo", "bar"])
self.step_mgr()
# Then decref "bar" so that it gets marked as stopping.
self.mgr.decref("bar", async=True)
self.step_mgr()
self.assertEqual(self.mgr.stopping_objects_by_id,
{"bar": set(self.created_refs["bar"])})
# Return mix of expected and unexpected ipsets.
m_list_ipsets.return_value = [
"not-felix-foo",
"felix-6-foo",
"felix-6-bazzle",
"felix-4-foo",
"felix-4-bar",
"felix-4-baz",
"felix-4-biff",
]
m_check_call.side_effect = iter([
# Exception on any individual call should be ignored.
FailedSystemCall("Dummy", [], None, None, None),
None,
None,
None,
])
self.mgr.cleanup(async=True)
self.step_mgr()
# Explicitly check that exactly the right delete calls were made.
# assert_has_calls would ignore extra calls.
self.assertEqual(
sorted(m_check_call.mock_calls),
sorted([
call(["ipset", "destroy", "felix-4-biff"]),
call(["ipset", "destroy", "felix-4-baz"]),
]))
def test_replace_members_delete_fails(self, m_check_call):
m_check_call.side_effect = iter(
[FailedSystemCall("Blah", [], 1, None, "err"), None, None, None])
self.ipset.replace_members(set(["10.0.0.1"]))
exp_calls = [
call(["ipset", "destroy", "foo-tmp"]),
call(['ipset', 'list', 'foo-tmp']),
call(['ipset', 'list', 'foo']),
call(["ipset", "restore"],
input_str='create foo-tmp hash:ip family inet '
'maxelem 1048576 --exist\n'
'flush foo-tmp\n'
'add foo-tmp 10.0.0.1\n'
'swap foo foo-tmp\n'
'destroy foo-tmp\n'
'COMMIT\n')
]
self.assertEqual(m_check_call.mock_calls, exp_calls)
def test_exception(self):
"""
Test exception when adding ipset value.
"""
description = "description"
suffix = "suffix"
rule_list = [{'cidr': "1.2.3.4/24"}]
self.create_ipsets("inet")
with mock.patch('calico.felix.test.stub_ipsets.add',
side_effect=FailedSystemCall("oops", [], 1, "", "")):
frules.update_ipsets(IPV4, description, suffix, rule_list,
"ipset_addr", "ipset_port", "ipset_icmp",
"tmp_ipset_addr", "tmp_ipset_port",
"tmp_ipset_icmp")
stub_ipsets.check_state(expected_ipsets)
def test_ensure_rule_inserted(self):
fragment = "FOO --jump DROP"
with patch.object(self.ipt, "_execute_iptables") as m_exec:
m_exec.side_effect = iter([
FailedSystemCall("Message", [], 1, "", "line 2 failed"), None,
None
])
self.ipt.ensure_rule_inserted(fragment, async=True)
self.step_actor(self.ipt)
self.assertEqual(m_exec.mock_calls, [
call([
"*filter", "--delete FOO --jump DROP",
"--insert FOO --jump DROP", "COMMIT"
],
fail_log_level=logging.DEBUG),
call(["*filter", "--insert FOO --jump DROP", "COMMIT"]),
])
self.assertTrue(fragment in self.ipt._inserted_rule_fragments)
def test_delete_transient_ipt_error(self):
"""
Test that the dirty flag is left set if a delete fails. Future
deletes that would normally be squashed trigger a retry.
"""
# First delete fails.
self.rules.on_profile_update(RULES_1, async=True)
self.step_actor(self.rules)
self._process_ipset_refs(set(["dst-tag", "src-tag"]))
self.m_ipt_updater.delete_chains.side_effect = \
FailedSystemCall("fail", ["foo"], 1, "", "")
self.rules.on_profile_update(None, async=True)
real_discard_all = self.rules._ipset_refs.discard_all
with patch.object(self.rules._ipset_refs,
"discard_all",
wraps=real_discard_all) as m_discard:
self.step_actor(self.rules)
# Failure should prevent freeing of ipset refs.
self.assertEqual(m_discard.mock_calls, [])
self._process_ipset_refs(set([]))
self.m_ipt_updater.delete_chains.assert_called_once_with(set(
RULES_1_CHAINS.keys()),
async=False)
# Failure should leave ProfileRules dirty.
self.assertTrue(self.rules._dirty)
# Update should trigger retry even though there was no change
# of data.
self.m_ipt_updater.reset_mock()
self.m_ipt_updater.delete_chains.side_effect = None
self.rules.on_profile_update(None, async=True)
with patch.object(self.rules._ipset_refs,
"discard_all",
wraps=real_discard_all) as m_discard:
self.step_actor(self.rules)
self.assertEqual(m_discard.mock_calls, [call()])
self._process_ipset_refs(set([]))
self.m_ipt_updater.delete_chains.assert_called_once_with(set(
RULES_1_CHAINS.keys()),
async=False)
# Successful delete leaves profile clean.
self.assertFalse(self.rules._dirty)
def _handle_rule(self, rule):
splits = rule.split(" ")
ipt_op = splits[0]
chain = splits[1]
_log.debug("Rule op: %s, chain name: %s", ipt_op, chain)
if ipt_op in ("--append", "-A", "--insert", "-I"):
self.assert_chain_declared(chain, ipt_op)
if ipt_op in ("--append", "-A"):
self.new_contents[chain].append(rule)
else:
self.new_contents[chain].insert(0, rule)
m = re.search(r'(?:--jump|-j|--goto|-g)\s+(\S+)', rule)
if m:
action = m.group(1)
_log.debug("Action %s", action)
if action not in STANDARD_ACTIONS:
# Assume a dependent chain.
self.new_dependencies[chain].add(action)
elif ipt_op in ("--delete-chain", "-X"):
self.assert_chain_declared(chain, ipt_op)
del self.new_contents[chain]
del self.new_dependencies[chain]
elif ipt_op in ("--flush", "-F"):
self.assert_chain_declared(chain, ipt_op)
self.new_contents[chain] = []
self.new_dependencies[chain] = set()
elif ipt_op in ("--delete", "-D"):
self.assert_chain_declared(chain, ipt_op)
for rule in self.new_contents.get(chain, []):
rule_fragment = " ".join(splits[1:])
if rule.endswith(rule_fragment):
self.new_contents[chain].remove(rule)
break
else:
raise FailedSystemCall("Delete for non-existent rule", [], 1,
"", "line 2 failed")
else:
raise AssertionError("Unknown operation %s; was expecting "
"'--append|flush|delete|...' " % ipt_op)
class TestIpset(BaseTestCase):
def setUp(self):
super(TestIpset, self).setUp()
self.ipset = Ipset("foo", "foo-tmp", "inet")
@patch("calico.felix.futils.check_call", autospec=True)
def test_replace_members(self, m_check_call):
self.ipset.replace_members(set(["10.0.0.1"]))
exp_calls = [
call(["ipset", "destroy", "foo-tmp"]),
call(["ipset", "list", "foo"]),
call(["ipset", "restore"],
input_str='create foo-tmp hash:ip family inet '
'maxelem 1048576 --exist\n'
'flush foo-tmp\n'
'add foo-tmp 10.0.0.1\n'
'swap foo foo-tmp\n'
'destroy foo-tmp\n'
'COMMIT\n')
]
self.assertEqual(m_check_call.mock_calls, exp_calls)
@patch("calico.felix.futils.check_call", autospec=True)
def test_replace_members_delete_fails(self, m_check_call):
m_check_call.side_effect = iter(
[FailedSystemCall("Blah", [], 1, None, "err"), None, None, None])
self.ipset.replace_members(set(["10.0.0.1"]))
exp_calls = [
call(["ipset", "destroy", "foo-tmp"]),
call(['ipset', 'list', 'foo-tmp']),
call(['ipset', 'list', 'foo']),
call(["ipset", "restore"],
input_str='create foo-tmp hash:ip family inet '
'maxelem 1048576 --exist\n'
'flush foo-tmp\n'
'add foo-tmp 10.0.0.1\n'
'swap foo foo-tmp\n'
'destroy foo-tmp\n'
'COMMIT\n')
]
self.assertEqual(m_check_call.mock_calls, exp_calls)
@patch("calico.felix.futils.check_call", autospec=True)
def test_apply_changes(self, m_check_call):
added = set(["10.0.0.2"])
removed = set(["10.0.0.1"])
self.ipset.apply_changes(added, removed)
self.assertEqual(m_check_call.mock_calls, [
call(["ipset", "restore"],
input_str='del foo 10.0.0.1\n'
'add foo 10.0.0.2\n'
'COMMIT\n')
])
@patch("calico.felix.futils.check_call",
autospec=True,
side_effect=FailedSystemCall("Blah", [], None, None, "err"))
def test_apply_changes_err(self, m_check_call):
# First call to update_members will fail, leading to a retry.
added = set(["10.0.0.2"])
removed = set(["10.0.0.1"])
self.assertRaises(FailedSystemCall, self.ipset.apply_changes, added,
removed)
@patch("calico.felix.futils.check_call", autospec=True)
def test_ensure_exists(self, m_check_call):
self.ipset.ensure_exists()
m_check_call.assert_called_once_with(
["ipset", "restore"],
input_str='create foo hash:ip family inet maxelem 1048576 --exist\n'
'COMMIT\n')
@patch("calico.felix.futils.call_silent", autospec=True)
def test_delete(self, m_call_silent):
self.ipset.delete()
self.assertEqual(m_call_silent.mock_calls, [
call(["ipset", "destroy", "foo"]),
call(["ipset", "destroy", "foo-tmp"]),
])
@patch("calico.felix.futils.check_call", autospec=True)
def test_list_ipset_names(self, m_check_call):
m_check_call.return_value = CommandOutput(IPSET_LIST_OUTPUT, "")
self.assertEqual(list_ipset_names(),
['felix-v4-calico_net', 'felix-v6-calico_net'])
def test_sync_to_ipset(self):
members1 = ["1.2.3.4", "2.3.4.5"]
members2 = ["1.2.3.6", "2.3.4.5"]
# Each call to replace members causes a full update.
_log.info("Calling replace_members() once...")
self.actor.replace_members(members1, async=True)
self.step_actor(self.actor)
self.ipset.replace_members.assert_called_once_with(set(members1))
self.assertFalse(self.ipset.apply_changes.called)
self.assertEqual(self.actor.members, set(members1))
self.assertFalse(self.actor._force_reprogram)
self.ipset.reset_mock()
_log.info("Calling replace_members() second time..")
self.actor.replace_members(members2, async=True)
self.step_actor(self.actor)
self.ipset.replace_members.assert_called_once_with(set(members2))
self.assertFalse(self.ipset.apply_changes.called)
self.assertEqual(self.actor.members, set(members2))
self.assertFalse(self.actor._force_reprogram)
self.ipset.reset_mock()
_log.info("Calling replace_members() third time with mixed "
"add/remove..")
self.actor.add_members(["5.6.7.8"], async=True) # Ignored
self.actor.remove_members(["1.2.3.6"], async=True)
self.actor.replace_members(members1, async=True)
self.actor.add_members(["6.7.8.9"], async=True) # Should apply
self.actor.remove_members(["2.3.4.5"], async=True)
self.step_actor(self.actor)
self.ipset.replace_members.assert_called_once_with(
set([
"6.7.8.9",
"1.2.3.4",
]))
self.assertFalse(self.ipset.apply_changes.called)
self.assertEqual(self.actor.members, set([
"6.7.8.9",
"1.2.3.4",
]))
self.assertFalse(self.actor._force_reprogram)
self.ipset.reset_mock()
_log.info("Calling add/remove.")
self.actor.add_members(["5.6.7.8"], async=True)
self.actor.remove_members(["1.2.3.4"], async=True)
self.step_actor(self.actor)
self.assertFalse(self.ipset.replace_members.called)
self.assertEqual(self.ipset.apply_changes.mock_calls,
[call(set(["5.6.7.8"]), set(["1.2.3.4"]))])
self.assertEqual(self.actor.members, set([
"6.7.8.9",
"5.6.7.8",
]))
self.assertFalse(self.actor._force_reprogram)
self.ipset.reset_mock()
_log.info("Calling add/remove with failure.")
self.ipset.apply_changes.side_effect = FailedSystemCall(
"", [], 1, "", "")
self.actor.add_members(["6.7.8.10"], async=True)
self.actor.remove_members(["5.6.7.8"], async=True)
self.step_actor(self.actor)
self.assertEqual(self.ipset.replace_members.mock_calls,
[call(set(["6.7.8.9", "6.7.8.10"]))])
self.assertEqual(self.actor.members, set(["6.7.8.9", "6.7.8.10"]))
self.assertFalse(self.actor._force_reprogram)
self.ipset.reset_mock()
class TestIpset(BaseTestCase):
def setUp(self):
super(TestIpset, self).setUp()
self.ipset = Ipset("foo", "foo-tmp", "inet")
@patch("calico.felix.futils.check_call", autospec=True)
def test_replace_members(self, m_check_call):
self.ipset.replace_members(set(["10.0.0.1"]))
m_check_call.assert_called_once_with(
["ipset", "restore"],
input_str='create foo hash:ip family inet --exist\n'
'create foo-tmp hash:ip family inet --exist\n'
'flush foo-tmp\n'
'add foo-tmp 10.0.0.1\n'
'swap foo foo-tmp\n'
'destroy foo-tmp\n'
'COMMIT\n')
@patch("calico.felix.futils.check_call", autospec=True)
def test_update_members(self, m_check_call):
old = set(["10.0.0.2"])
new = set(["10.0.0.1", "10.0.0.2"])
self.ipset.update_members(old, new)
old = set(["10.0.0.1", "10.0.0.2"])
new = set(["10.0.0.1", "1.2.3.4"])
self.ipset.update_members(old, new)
calls = [
call(["ipset", "restore"], input_str='add foo 10.0.0.1\nCOMMIT\n'),
call(["ipset", "restore"],
input_str='del foo 10.0.0.2\n'
'add foo 1.2.3.4\n'
'COMMIT\n')
]
self.assertEqual(m_check_call.call_count, 2)
m_check_call.assert_has_calls(calls)
@patch("calico.felix.futils.check_call",
autospec=True,
side_effect=iter(
[FailedSystemCall("Blah", [], None, None, "err"), None]))
def test_update_members_err(self, m_check_call):
# First call to update_members will fail, leading to a retry.
old = set(["10.0.0.2"])
new = set(["10.0.0.1"])
self.ipset.update_members(old, new)
calls = [
call(["ipset", "restore"],
input_str='del foo 10.0.0.2\n'
'add foo 10.0.0.1\n'
'COMMIT\n'),
call(["ipset", "restore"],
input_str='create foo hash:ip family inet --exist\n'
'create foo-tmp hash:ip family inet --exist\n'
'flush foo-tmp\n'
'add foo-tmp 10.0.0.1\n'
'swap foo foo-tmp\n'
'destroy foo-tmp\n'
'COMMIT\n')
]
self.assertEqual(m_check_call.call_count, 2)
m_check_call.assert_has_calls(calls)
@patch("calico.felix.futils.check_call", autospec=True)
def test_ensure_exists(self, m_check_call):
self.ipset.ensure_exists()
m_check_call.assert_called_once_with(
["ipset", "restore"],
input_str='create foo hash:ip family inet --exist\n'
'COMMIT\n')
@patch("calico.felix.futils.call_silent", autospec=True)
def test_delete(self, m_call_silent):
self.ipset.delete()
self.assertEqual(m_call_silent.mock_calls, [
call(["ipset", "destroy", "foo"]),
call(["ipset", "destroy", "foo-tmp"]),
])
def test_on_interface_update_v6(self):
combined_id = EndpointId("host_id", "orchestrator_id",
"workload_id", "endpoint_id")
ip_type = futils.IPV6
local_ep = self.get_local_endpoint(combined_id, ip_type)
ips = ["1234::5678"]
iface = "tapabcdef"
data = {
'state': "active",
'endpoint': "endpoint_id",
'mac': stub_utils.get_mac(),
'name': iface,
'ipv6_nets': ips,
'profile_ids': ["prof1"]
}
# We can only get on_interface_update calls after the first
# on_endpoint_update, so trigger that.
with nested(
mock.patch('calico.felix.devices.set_routes'),
mock.patch('calico.felix.devices.configure_interface_ipv6'),
mock.patch('calico.felix.devices.interface_up'),
) as [m_set_routes, m_conf, m_iface_up]:
m_iface_up.return_value = False
local_ep.on_endpoint_update(data, async=True)
self.step_actor(local_ep)
self.assertEqual(local_ep._mac, data['mac'])
m_conf.assert_called_once_with(iface, None)
m_set_routes.assert_called_once_with(ip_type,
set(ips),
iface,
data['mac'],
reset_arp=False)
# Now pretend to get an interface update - does all the same work.
with mock.patch('calico.felix.devices.set_routes') as m_set_routes:
with mock.patch('calico.felix.devices.'
'configure_interface_ipv6') as m_conf:
local_ep.on_interface_update(True, async=True)
self.step_actor(local_ep)
m_conf.assert_called_once_with(iface, None)
m_set_routes.assert_called_once_with(ip_type,
set(ips),
iface,
data['mac'],
reset_arp=False)
self.assertTrue(local_ep._device_in_sync)
# Now cover the error cases...
with mock.patch('calico.felix.devices.'
'configure_interface_ipv6') as m_conf:
with mock.patch('calico.felix.devices.'
'interface_exists') as ifce_exists:
with mock.patch('calico.felix.devices.'
'interface_up') as ifce_up:
# Cycle through all the possibilities for the state.
ifce_exists.side_effect = [True, False, True]
ifce_up.side_effect = [True, False]
m_conf.side_effect = FailedSystemCall("", [], 1, "", "")
local_ep.on_interface_update(False, async=True)
self.step_actor(local_ep)
local_ep.on_interface_update(True, async=True)
self.step_actor(local_ep)
local_ep.on_interface_update(True, async=True)
self.step_actor(local_ep)
self.assertFalse(local_ep._device_in_sync)