def test_nonzero_default_delays(self, mocked_delay_node):
topology = {
"cluster_type": "replset",
"mongod": [
{"public_ip": "1.2.3.4", "private_ip": "10.2.0.1"},
{"public_ip": "2.3.4.5", "private_ip": "10.2.0.2"},
{"public_ip": "3.4.5.6", "private_ip": "10.2.0.3"},
],
}
delay_config = {"default": {"delay_ms": 100, "jitter_ms": 10}, "edges": []}
DelayGraph.client_node = mocked_delay_node.return_value
delay_graph = DelayGraph(topology, delay_config)
self.assertEqual(len(delay_graph.graph), 4)
delay_spec = DelaySpec({"delay_ms": 100, "jitter_ms": 10})
self.assertEqual(delay_graph.default_delay.delay_ms, delay_spec.delay_ms)
self.assertEqual(delay_graph.default_delay.jitter_ms, delay_spec.jitter_ms)
# Each IP gets called thrice: once for each of the other nodes.
expected = [
call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True),
call("workload_client", delay_graph.default_delay, defer_to_edgewise=True),
call("workload_client", delay_graph.default_delay, defer_to_edgewise=True),
call("workload_client", delay_graph.default_delay, defer_to_edgewise=True),
]
self.assertEqual(mocked_delay_node.return_value.add.call_count, len(expected))
mocked_delay_node.return_value.add.assert_has_calls(expected, any_order=True)
def test_parse_standalone(self):
topology = {"cluster_type": "standalone", "public_ip": "1.2.3.4", "private_ip": "10.2.0.1"}
delay_graph = DelayGraph(topology, BASIC_DELAY_CONFIG)
expected = ["10.2.0.1", "workload_client"]
for private_ip in delay_graph.graph:
self.assertTrue(private_ip in expected)
self.assertEqual(len(expected), len(delay_graph.graph))
def test_zero_edgewise_delay(self, mocked_delay_node):
topology = {
"cluster_type": "replset",
"mongod": [
{"public_ip": "1.2.3.4", "private_ip": "10.2.0.1"},
{"public_ip": "2.3.4.5", "private_ip": "10.2.0.2"},
{"public_ip": "3.4.5.6", "private_ip": "10.2.0.3"},
],
}
delay_config = {
"default": {"delay_ms": 100, "jitter_ms": 10},
"edges": [
{"node1": "10.2.0.1", "node2": "10.2.0.2", "delay": {"delay_ms": 0, "jitter_ms": 0}}
],
}
DelayGraph.client_node = mocked_delay_node.return_value
delay_graph = DelayGraph(topology, delay_config)
self.assertEqual(len(delay_graph.graph), 4)
expected_edge_spec = EdgeSpec(
{"node1": "10.2.0.1", "node2": "10.2.0.2", "delay": {"delay_ms": 0, "jitter_ms": 0}}
)
expected_delay_spec = expected_edge_spec.delay
actual_edge_spec = delay_graph.edgewise_delays[0]
actual_delay_spec = actual_edge_spec.delay
self.assertEqual(actual_edge_spec.node1, expected_edge_spec.node1)
self.assertEqual(actual_edge_spec.node2, expected_edge_spec.node2)
self.assertEqual(actual_delay_spec.delay_ms, expected_delay_spec.delay_ms)
self.assertEqual(actual_delay_spec.jitter_ms, expected_delay_spec.jitter_ms)
# Each IP gets called once for each of the other nodes and the workload client.
# The nodes along the edge each get called an additional time.
expected = [
call("10.2.0.1", actual_delay_spec),
call("10.2.0.2", actual_delay_spec),
call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.1", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.2", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True),
call("10.2.0.3", delay_graph.default_delay, defer_to_edgewise=True),
call("workload_client", delay_graph.default_delay, defer_to_edgewise=True),
call("workload_client", delay_graph.default_delay, defer_to_edgewise=True),
call("workload_client", delay_graph.default_delay, defer_to_edgewise=True),
]
self.assertEqual(mocked_delay_node.return_value.add.call_count, len(expected))
mocked_delay_node.return_value.add.assert_has_calls(expected, any_order=True)
def test_parse_replset(self):
topology = {
"cluster_type": "replset",
"mongod": [
{"public_ip": "1.2.3.4", "private_ip": "10.2.0.1"},
{"public_ip": "2.3.4.5", "private_ip": "10.2.0.2"},
{"public_ip": "3.4.5.6", "private_ip": "10.2.0.3"},
],
}
delay_graph = DelayGraph(topology, BASIC_DELAY_CONFIG)
expected = ["10.2.0.1", "10.2.0.2", "10.2.0.3", "workload_client"]
for private_ip in delay_graph.graph:
self.assertTrue(private_ip in expected)
self.assertEqual(len(expected), len(delay_graph.graph))
def test_validate_delays(self):
blacklisted_configs = ["mongodb_setup.atlas.yml"] # Some configs don't have topologies.
directory = whereami.dsi_repo_path("configurations", "mongodb_setup")
errors = []
# There are a few files that aren't configuration files.
names = [name for name in os.listdir(directory) if name.startswith("mongodb_setup")]
# We need references to provisioning output
infrastructure_provisioning = whereami.dsi_repo_path(
"docs/config-specs/infrastructure_provisioning.out.yml"
)
with copied_file(infrastructure_provisioning, "infrastructure_provisioning.out.yml"):
for conf_name in names:
if conf_name in blacklisted_configs:
continue
with copied_file(os.path.join(directory, conf_name), "mongodb_setup.yml"):
config = ConfigDict("mongodb_setup")
config.load()
topologies = config["mongodb_setup"]["topology"]
network_delays = config["mongodb_setup"]["network_delays"]
delay_configs = network_delays["clusters"]
try:
# The DelayNodes throw exceptions when given bad delays, so we
# can validate the configuration by simply constructing a DelayGraph
# pylint: disable=unused-variable
DelayGraph.client_ip = config["infrastructure_provisioning"]["out"][
"workload_client"
][0]["private_ip"]
version_flag = str_to_version_flag(
network_delays.get("version_flag", "default")
)
DelayGraph.client_node = DelayNode(version_flag)
delays = DelayGraph.from_topologies(topologies, delay_configs, version_flag)
# pylint: disable=broad-except
except Exception as e:
errors.append(e)
# Reset the delay graph's client variables.
DelayGraph.client_node = DelayNode()
DelayGraph.client_ip = "workload_client"
self.assertEqual(errors, [])
def test_parse_sharded(self):
topology = {
"cluster_type": "sharded_cluster",
"configsvr": [
{"public_ip": "1.2.3.4", "private_ip": "10.2.0.1"},
{"public_ip": "2.3.4.5", "private_ip": "10.2.0.2"},
{"public_ip": "3.4.5.6", "private_ip": "10.2.0.3"},
],
"mongos": [
{"public_ip": "6.7.8.9", "private_ip": "10.2.0.4"},
{"public_ip": "7.8.9.10", "private_ip": "10.2.0.5"},
{"public_ip": "8.9.10.11", "private_ip": "10.2.0.6"},
],
"shard": [
{
"cluster_type": "replset",
"mongod": [
{"public_ip": "9.10.11.12", "private_ip": "10.2.0.7"},
{"public_ip": "10.11.12.13", "private_ip": "10.2.0.8"},
{"public_ip": "11.12.13.14", "private_ip": "10.2.0.9"},
],
},
{
"cluster_type": "replset",
"mongod": [
{"public_ip": "12.13.14.15", "private_ip": "10.2.0.10"},
{"public_ip": "13.14.15.16", "private_ip": "10.2.0.11"},
{"public_ip": "14.15.16.17", "private_ip": "10.2.0.12"},
],
},
{
"cluster_type": "replset",
"mongod": [
{"public_ip": "15.16.17.18", "private_ip": "10.2.0.13"},
{"public_ip": "16.17.18.19", "private_ip": "10.2.0.14"},
{"public_ip": "17.18.19.20", "private_ip": "10.2.0.15"},
],
},
],
}
delay_graph = DelayGraph(topology, BASIC_DELAY_CONFIG)
expected = ["10.2.0.{num}".format(num=i) for i in range(1, 16)]
expected.append("workload_client")
for private_ip in delay_graph.graph:
self.assertTrue(private_ip in expected)
self.assertEqual(len(expected), len(delay_graph.graph))
def parse_cluster(self):
"""Create cluster for each topology and delay"""
inf_prov_out = self.config["infrastructure_provisioning"]
client_config = inf_prov_out["out"]["workload_client"][0]
client_ip = client_config["private_ip"]
DelayGraph.client_ip = client_ip
topologies = self.mongodb_setup.get("topology", [])
network_delays = self.mongodb_setup.get("network_delays", {})
delay_configs = network_delays.get("clusters", [])
version_flag = str_to_version_flag(
network_delays.get("version_flag", "default"))
DelayGraph.client_node = DelayNode(version_flag)
delays = DelayGraph.from_topologies(topologies, delay_configs,
version_flag)
client_config = ClientConfig(self.config)
self.client = Client(client_config)
# pylint: disable=consider-using-enumerate
for i in range(len(topologies)):
self.clusters.append(
mongodb_cluster.create_cluster(topologies[i], delays[i],
self.config))