def test_negative_jitter(self):
delay = DelayNode()
nonzero_delay_spec_1 = DelaySpec({"delay_ms": 10, "jitter_ms": 3})
nonzero_delay_spec_2 = DelaySpec({"delay_ms": 10, "jitter_ms": 10})
negative_jitter_spec = DelaySpec({"delay_ms": 50, "jitter_ms": -5})
delay.add("fake_ip_1", nonzero_delay_spec_1)
delay.add("fake_ip_2", nonzero_delay_spec_2)
self.assertRaises(DelayError, delay.add, "fake_ip_3", negative_jitter_spec)
def test_already_defined_ip(self):
delay = DelayNode()
nonzero_delay_spec_1 = DelaySpec({"delay_ms": 700, "jitter_ms": 10})
nonzero_delay_spec_2 = DelaySpec({"delay_ms": 300, "jitter_ms": 0})
nonzero_delay_spec_3 = DelaySpec({"delay_ms": 200, "jitter_ms": 0})
delay.add("fake_ip_1", nonzero_delay_spec_1)
delay.add("fake_ip_2", nonzero_delay_spec_2)
self.assertRaises(DelayError, delay.add, "fake_ip_1", nonzero_delay_spec_3)
def test_one_nonzero_delay(self):
node = DelayNode()
zero_delay_spec = DelaySpec({"delay_ms": 0, "jitter_ms": 0})
nonzero_delay_spec = DelaySpec({"delay_ms": 100, "jitter_ms": 5})
node.add("fake_ip_str_1", zero_delay_spec)
node.add("fake_ip_str_2", nonzero_delay_spec)
node.add("fake_ip_str_3", zero_delay_spec)
expected = [
["bash", "-c", "'sudo tc qdisc add dev eth0 root handle 1: htb default 1'"],
[
"bash",
"-c",
"'sudo tc class add dev eth0 parent 1: classid 1:1 htb rate 100tbit prio 0'",
],
[
"bash",
"-c",
"'sudo tc class add dev eth0 parent 1: classid 1:2 htb rate 100tbit prio 0'",
],
["bash", "-c", "'sudo tc qdisc add dev eth0 parent 1:2 netem delay 0ms 0ms'"],
[
"bash",
"-c",
"'sudo tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dst fake_ip_str_1 flowid 1:2'",
],
[
"bash",
"-c",
"'sudo tc class add dev eth0 parent 1: classid 1:3 htb rate 100tbit prio 0'",
],
["bash", "-c", "'sudo tc qdisc add dev eth0 parent 1:3 netem delay 100ms 5ms'"],
[
"bash",
"-c",
"'sudo tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dst fake_ip_str_2 flowid 1:3'",
],
[
"bash",
"-c",
"'sudo tc class add dev eth0 parent 1: classid 1:4 htb rate 100tbit prio 0'",
],
["bash", "-c", "'sudo tc qdisc add dev eth0 parent 1:4 netem delay 0ms 0ms'"],
[
"bash",
"-c",
"'sudo tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dst fake_ip_str_3 flowid 1:4'",
],
]
actual = node.generate_delay_commands()
self.assertEqual(len(actual), len(expected))
for i, expect in enumerate(expected):
self.assertEqual(actual[i].command, expect)
def test_nonzero_default_delays(self, mocked_delay_node):
topology = {
"cluster_type": "replset",
"mongod": [
{"public_ip": "1.2.3.4", "private_ip": "10.2.0.1"},
{"public_ip": "2.3.4.5", "private_ip": "10.2.0.2"},
{"public_ip": "3.4.5.6", "private_ip": "10.2.0.3"},
],
}
delay_config = {"default": {"delay_ms": 100, "jitter_ms": 10}, "edges": []}
DelayGraph.client_node = mocked_delay_node.return_value
delay_graph = DelayGraph(topology, delay_config)
self.assertEqual(len(delay_graph.graph), 4)
delay_spec = DelaySpec({"delay_ms": 100, "jitter_ms": 10})
self.assertEqual(delay_graph.default_delay.delay_ms, delay_spec.delay_ms)
self.assertEqual(delay_graph.default_delay.jitter_ms, delay_spec.jitter_ms)
# Each IP gets called thrice: once for each of the other nodes.
expected = [
call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True),
call("workload_client", delay_graph.default_delay, defer_to_edgewise=True),
call("workload_client", delay_graph.default_delay, defer_to_edgewise=True),
call("workload_client", delay_graph.default_delay, defer_to_edgewise=True),
]
self.assertEqual(mocked_delay_node.return_value.add.call_count, len(expected))
mocked_delay_node.return_value.add.assert_has_calls(expected, any_order=True)
def test_assert_kernel_fails_m60_like(self):
mocked_host = MagicMock()
def mocked_run(command):
if command == ["bash", "-c", '\'uname -r | cut -d "." -f 1 | grep -q "3"\'']:
return False
return True
mocked_host.run = mocked_run
nonzero_delay_spec_1 = DelaySpec({"delay_ms": 400, "jitter_ms": 10})
nonzero_delay_spec_2 = DelaySpec({"delay_ms": 100, "jitter_ms": 50})
delay = DelayNode(VersionFlag.M60_LIKE)
delay.add("fake_ip_1", nonzero_delay_spec_1)
delay.add("fake_ip_2", nonzero_delay_spec_2)
self.assertRaises(DelayError, delay.reset_delays, mocked_host)
def test_zero_delays(self):
zero_delay_spec = DelaySpec({"delay_ms": 0, "jitter_ms": 0})
node = DelayNode()
node.add("fake_ip_str_1", zero_delay_spec)
node.add("fake_ip_str_2", zero_delay_spec)
node.add("fake_ip_str_3", zero_delay_spec)
expected = []
actual = node.generate_delay_commands()
self.assertEqual(len(actual), len(expected))
for i, expect in enumerate(expected):
self.assertEqual(actual[i].command, expect)
def test_establish_delays_fails(self):
mocked_host = MagicMock()
def mocked_run(command):
if command == [
"bash",
"-c",
"'sudo tc qdisc add dev eth0 root handle 1: htb default 1'",
]:
return False
return True
mocked_host.run = mocked_run
nonzero_delay_spec_1 = DelaySpec({"delay_ms": 700, "jitter_ms": 10})
nonzero_delay_spec_2 = DelaySpec({"delay_ms": 300, "jitter_ms": 0})
delay = DelayNode()
delay.add("fake_ip_1", nonzero_delay_spec_1)
delay.add("fake_ip_2", nonzero_delay_spec_2)
self.assertRaises(DelayError, delay.establish_delays, mocked_host)
def test_establish_delays_runs_commands(self):
mocked_host = MagicMock()
mocked_host.run = MagicMock()
mocked_host.run.return_value = True
nonzero_delay_spec_1 = DelaySpec({"delay_ms": 700, "jitter_ms": 10})
nonzero_delay_spec_2 = DelaySpec({"delay_ms": 300, "jitter_ms": 0})
delay = DelayNode()
delay.add("fake_ip_1", nonzero_delay_spec_1)
delay.add("fake_ip_2", nonzero_delay_spec_2)
delay.reset_delays(mocked_host)
delay.establish_delays(mocked_host)
expected_calls = [
call(["bash", "-c", '\'uname -r | cut -d "." -f 1 | grep -q "4"\'']),
call(
[
"bash",
"-c",
'\'yum info iproute | grep "Version" | cut -d ":" -f 2 | cut -d " " -f 2 | cut -d "." -f 1 | grep -q "4"\'',
]
),
call(["bash", "-c", "'sudo tc qdisc del dev eth0 root'"]),
call(["bash", "-c", "'sudo tc qdisc add dev eth0 root handle 1: htb default 1'"]),
call(
[
"bash",
"-c",
"'sudo tc class add dev eth0 parent 1: classid 1:1 htb rate 100tbit prio 0'",
]
),
call(
[
"bash",
"-c",
"'sudo tc class add dev eth0 parent 1: classid 1:2 htb rate 100tbit prio 0'",
]
),
call(["bash", "-c", "'sudo tc qdisc add dev eth0 parent 1:2 netem delay 700ms 10ms'"]),
call(
[
"bash",
"-c",
"'sudo tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dst fake_ip_1 flowid 1:2'",
]
),
call(
[
"bash",
"-c",
"'sudo tc class add dev eth0 parent 1: classid 1:3 htb rate 100tbit prio 0'",
]
),
call(["bash", "-c", "'sudo tc qdisc add dev eth0 parent 1:3 netem delay 300ms 0ms'"]),
call(
[
"bash",
"-c",
"'sudo tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dst fake_ip_2 flowid 1:3'",
]
),
]
self.assertEqual(mocked_host.run.call_count, len(expected_calls))
mocked_host.run.assert_has_calls(expected_calls)