def test_negative_jitter(self): delay = DelayNode() nonzero_delay_spec_1 = DelaySpec({"delay_ms": 10, "jitter_ms": 3}) nonzero_delay_spec_2 = DelaySpec({"delay_ms": 10, "jitter_ms": 10}) negative_jitter_spec = DelaySpec({"delay_ms": 50, "jitter_ms": -5}) delay.add("fake_ip_1", nonzero_delay_spec_1) delay.add("fake_ip_2", nonzero_delay_spec_2) self.assertRaises(DelayError, delay.add, "fake_ip_3", negative_jitter_spec)
def test_already_defined_ip(self): delay = DelayNode() nonzero_delay_spec_1 = DelaySpec({"delay_ms": 700, "jitter_ms": 10}) nonzero_delay_spec_2 = DelaySpec({"delay_ms": 300, "jitter_ms": 0}) nonzero_delay_spec_3 = DelaySpec({"delay_ms": 200, "jitter_ms": 0}) delay.add("fake_ip_1", nonzero_delay_spec_1) delay.add("fake_ip_2", nonzero_delay_spec_2) self.assertRaises(DelayError, delay.add, "fake_ip_1", nonzero_delay_spec_3)
def test_one_nonzero_delay(self): node = DelayNode() zero_delay_spec = DelaySpec({"delay_ms": 0, "jitter_ms": 0}) nonzero_delay_spec = DelaySpec({"delay_ms": 100, "jitter_ms": 5}) node.add("fake_ip_str_1", zero_delay_spec) node.add("fake_ip_str_2", nonzero_delay_spec) node.add("fake_ip_str_3", zero_delay_spec) expected = [ ["bash", "-c", "'sudo tc qdisc add dev eth0 root handle 1: htb default 1'"], [ "bash", "-c", "'sudo tc class add dev eth0 parent 1: classid 1:1 htb rate 100tbit prio 0'", ], [ "bash", "-c", "'sudo tc class add dev eth0 parent 1: classid 1:2 htb rate 100tbit prio 0'", ], ["bash", "-c", "'sudo tc qdisc add dev eth0 parent 1:2 netem delay 0ms 0ms'"], [ "bash", "-c", "'sudo tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dst fake_ip_str_1 flowid 1:2'", ], [ "bash", "-c", "'sudo tc class add dev eth0 parent 1: classid 1:3 htb rate 100tbit prio 0'", ], ["bash", "-c", "'sudo tc qdisc add dev eth0 parent 1:3 netem delay 100ms 5ms'"], [ "bash", "-c", "'sudo tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dst fake_ip_str_2 flowid 1:3'", ], [ "bash", "-c", "'sudo tc class add dev eth0 parent 1: classid 1:4 htb rate 100tbit prio 0'", ], ["bash", "-c", "'sudo tc qdisc add dev eth0 parent 1:4 netem delay 0ms 0ms'"], [ "bash", "-c", "'sudo tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dst fake_ip_str_3 flowid 1:4'", ], ] actual = node.generate_delay_commands() self.assertEqual(len(actual), len(expected)) for i, expect in enumerate(expected): self.assertEqual(actual[i].command, expect)
def test_nonzero_default_delays(self, mocked_delay_node): topology = { "cluster_type": "replset", "mongod": [ {"public_ip": "1.2.3.4", "private_ip": "10.2.0.1"}, {"public_ip": "2.3.4.5", "private_ip": "10.2.0.2"}, {"public_ip": "3.4.5.6", "private_ip": "10.2.0.3"}, ], } delay_config = {"default": {"delay_ms": 100, "jitter_ms": 10}, "edges": []} DelayGraph.client_node = mocked_delay_node.return_value delay_graph = DelayGraph(topology, delay_config) self.assertEqual(len(delay_graph.graph), 4) delay_spec = DelaySpec({"delay_ms": 100, "jitter_ms": 10}) self.assertEqual(delay_graph.default_delay.delay_ms, delay_spec.delay_ms) self.assertEqual(delay_graph.default_delay.jitter_ms, delay_spec.jitter_ms) # Each IP gets called thrice: once for each of the other nodes. expected = [ call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True), call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True), call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True), call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True), call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True), call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True), call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True), call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True), call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True), call("workload_client", delay_graph.default_delay, defer_to_edgewise=True), call("workload_client", delay_graph.default_delay, defer_to_edgewise=True), call("workload_client", delay_graph.default_delay, defer_to_edgewise=True), ] self.assertEqual(mocked_delay_node.return_value.add.call_count, len(expected)) mocked_delay_node.return_value.add.assert_has_calls(expected, any_order=True)
def test_assert_kernel_fails_m60_like(self): mocked_host = MagicMock() def mocked_run(command): if command == ["bash", "-c", '\'uname -r | cut -d "." -f 1 | grep -q "3"\'']: return False return True mocked_host.run = mocked_run nonzero_delay_spec_1 = DelaySpec({"delay_ms": 400, "jitter_ms": 10}) nonzero_delay_spec_2 = DelaySpec({"delay_ms": 100, "jitter_ms": 50}) delay = DelayNode(VersionFlag.M60_LIKE) delay.add("fake_ip_1", nonzero_delay_spec_1) delay.add("fake_ip_2", nonzero_delay_spec_2) self.assertRaises(DelayError, delay.reset_delays, mocked_host)
def test_zero_delays(self): zero_delay_spec = DelaySpec({"delay_ms": 0, "jitter_ms": 0}) node = DelayNode() node.add("fake_ip_str_1", zero_delay_spec) node.add("fake_ip_str_2", zero_delay_spec) node.add("fake_ip_str_3", zero_delay_spec) expected = [] actual = node.generate_delay_commands() self.assertEqual(len(actual), len(expected)) for i, expect in enumerate(expected): self.assertEqual(actual[i].command, expect)
def test_establish_delays_fails(self): mocked_host = MagicMock() def mocked_run(command): if command == [ "bash", "-c", "'sudo tc qdisc add dev eth0 root handle 1: htb default 1'", ]: return False return True mocked_host.run = mocked_run nonzero_delay_spec_1 = DelaySpec({"delay_ms": 700, "jitter_ms": 10}) nonzero_delay_spec_2 = DelaySpec({"delay_ms": 300, "jitter_ms": 0}) delay = DelayNode() delay.add("fake_ip_1", nonzero_delay_spec_1) delay.add("fake_ip_2", nonzero_delay_spec_2) self.assertRaises(DelayError, delay.establish_delays, mocked_host)
def test_establish_delays_runs_commands(self): mocked_host = MagicMock() mocked_host.run = MagicMock() mocked_host.run.return_value = True nonzero_delay_spec_1 = DelaySpec({"delay_ms": 700, "jitter_ms": 10}) nonzero_delay_spec_2 = DelaySpec({"delay_ms": 300, "jitter_ms": 0}) delay = DelayNode() delay.add("fake_ip_1", nonzero_delay_spec_1) delay.add("fake_ip_2", nonzero_delay_spec_2) delay.reset_delays(mocked_host) delay.establish_delays(mocked_host) expected_calls = [ call(["bash", "-c", '\'uname -r | cut -d "." -f 1 | grep -q "4"\'']), call( [ "bash", "-c", '\'yum info iproute | grep "Version" | cut -d ":" -f 2 | cut -d " " -f 2 | cut -d "." -f 1 | grep -q "4"\'', ] ), call(["bash", "-c", "'sudo tc qdisc del dev eth0 root'"]), call(["bash", "-c", "'sudo tc qdisc add dev eth0 root handle 1: htb default 1'"]), call( [ "bash", "-c", "'sudo tc class add dev eth0 parent 1: classid 1:1 htb rate 100tbit prio 0'", ] ), call( [ "bash", "-c", "'sudo tc class add dev eth0 parent 1: classid 1:2 htb rate 100tbit prio 0'", ] ), call(["bash", "-c", "'sudo tc qdisc add dev eth0 parent 1:2 netem delay 700ms 10ms'"]), call( [ "bash", "-c", "'sudo tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dst fake_ip_1 flowid 1:2'", ] ), call( [ "bash", "-c", "'sudo tc class add dev eth0 parent 1: classid 1:3 htb rate 100tbit prio 0'", ] ), call(["bash", "-c", "'sudo tc qdisc add dev eth0 parent 1:3 netem delay 300ms 0ms'"]), call( [ "bash", "-c", "'sudo tc filter add dev eth0 protocol ip parent 1:0 prio 1 u32 match ip dst fake_ip_2 flowid 1:3'", ] ), ] self.assertEqual(mocked_host.run.call_count, len(expected_calls)) mocked_host.run.assert_has_calls(expected_calls)