def test_parses_result_with_one_member(self):
"""Test that parses result with one member."""
expected_node = {
"member_id":
"415090d15def9053",
"name":
"toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud",
"status":
"up",
"isLeader":
True,
"peerURLs":
"https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380",
"clientURLs":
"https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379",
}
mock_run_sync = _get_mock_run_sync(
return_value=b"""
415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true
""", # noqa: E501
)
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
gotten_result = controller.get_cluster_info()
mock_run_sync.assert_called_once()
assert len(gotten_result) == 1
assert expected_node["member_id"] in gotten_result
assert gotten_result[expected_node["member_id"]] == expected_node
def test_check_core_masters_in_sync_fail_heartbeat(self, mocked_sleep):
"""Should raise MysqlLegacyError if unable to get the heartbeat from the current master."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001"))
mock_cumin(self.mocked_transports, 0, retvals=[])
with pytest.raises(mysql_legacy.MysqlLegacyError, match="Unable to get heartbeat from master"):
self.mysql.check_core_masters_in_sync("eqiad", "codfw")
assert not mocked_sleep.called
def test_get_core_dbs_fail_sanity_check(self):
"""It should raise MysqlLegacyError if matching an invalid number of hosts when looking for masters."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001"))
with pytest.raises(mysql_legacy.MysqlLegacyError, match="Matched 1 masters, expected 12"):
self.mysql.get_core_dbs(datacenter="eqiad", replication_role="master")
assert self.mocked_remote.query.called
def test_check_core_masters_in_sync_not_in_sync(self, mocked_sleep):
"""Should raise MysqlLegacyError if a master is not in sync with the one in the other DC."""
hosts = NodeSet(EQIAD_CORE_MASTERS_QUERY)
self.mocked_remote.query.side_effect = [RemoteHosts(self.config, NodeSet(host)) for host in hosts] + [
RemoteHosts(self.config, NodeSet("db1001"))
] * 3
retvals = [[(host, b"2018-09-06T10:00:00.000000")] for host in hosts] # first heartbeat
retvals += [[("db1001", b"2018-09-06T10:00:00.000000")]] * 3 # 3 failed retries of second heartbeat
mock_cumin(self.mocked_transports, 0, retvals=retvals)
with pytest.raises(
mysql_legacy.MysqlLegacyError,
match=r"Heartbeat from master db1001 for section .* not yet in sync",
):
self.mysql.check_core_masters_in_sync("eqiad", "codfw")
assert mocked_sleep.called
def test_raises_if_more_than_one_node_is_used(self):
"""Test that raises if more than one node is used."""
nodes = RemoteHosts(config=mock.MagicMock(specset=Config),
hosts=NodeSet("test[0,1].local.host"))
with self.assertRaises(TooManyHosts):
EtcdctlController(remote_host=nodes)
def test_adds_the_member_if_not_there(self):
"""Test that adds the member if not there."""
new_member_fqdn = "i.already.exist"
new_member_peer_url = f"https://{new_member_fqdn}:1234"
expected_member_id = "1234556789012345"
mock_run_sync = _get_mock_run_sync(side_effect=[
b"""
415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true
""", # noqa: E501
b"""Added :)""",
f"""
415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=false
{expected_member_id}: name={new_member_fqdn} peerURLs={new_member_peer_url}
""".encode(), # noqa: E501
])
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
gotten_member_id = controller.ensure_node_exists(
new_member_fqdn=new_member_fqdn,
member_peer_url=new_member_peer_url,
)
_assert_called_with_single_param(param="add", mock_obj=mock_run_sync)
assert gotten_member_id == expected_member_id
def test_check_core_masters_in_sync_ok(self, mocked_sleep):
"""Should check that all core masters are in sync with the master in the other DC."""
hosts = NodeSet(EQIAD_CORE_MASTERS_QUERY)
self.mocked_remote.query.side_effect = [RemoteHosts(self.config, NodeSet(host)) for host in hosts] * 2
retvals = [[(host, b"2018-09-06T10:00:00.000000")] for host in hosts] # first heartbeat
retvals += [[(host, b"2018-09-06T10:00:01.000000")] for host in hosts] # second heartbeat
mock_cumin(self.mocked_transports, 0, retvals=retvals)
self.mysql.check_core_masters_in_sync("eqiad", "codfw")
assert not mocked_sleep.called
def setup_method(self, _, mocked_transports):
"""Setup the test environment."""
# pylint: disable=attribute-defined-outside-init
self.config = Config(get_fixture_path("remote", "config.yaml"))
self.mocked_transports = mocked_transports
self.mysql_remote_hosts = mysql_legacy.MysqlLegacyRemoteHosts(
RemoteHosts(self.config, NodeSet("host[1-9]"), dry_run=False)
)
self.expected = [(NodeSet("host1"), "output1")]
def test_get_core_masters_heartbeats_wrong_data(self):
"""Should raise MysqlLegacyError if unable to convert the heartbeat into a datetime."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001"))
mock_cumin(
self.mocked_transports,
0,
retvals=[[("db1001", b"2018-09-06-10:00:00.000000")]],
)
with pytest.raises(mysql_legacy.MysqlLegacyError, match="Unable to convert heartbeat"):
self.mysql.get_core_masters_heartbeats("eqiad", "codfw")
def test_verify_core_masters_readonly_fail(self):
"""Should raise MysqlLegacyError if some masters do not have the intended read-only value."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db10[01-11]"))
mock_cumin(
self.mocked_transports,
0,
retvals=[[("db1001", b"0"), ("db10[02-11]", b"1")]],
)
with pytest.raises(
mysql_legacy.MysqlLegacyError,
match="Verification failed that core DB masters",
):
self.mysql.verify_core_masters_readonly("eqiad", True)
def __new__(cls,
icinga_host: RemoteHosts,
*,
config_file: str = "/etc/icinga/icinga.cfg") -> "CommandFile":
"""Get the Icinga host command file where to write the commands and cache it.
Arguments:
icinga_host (spicerack.remote.RemoteHosts): the Icinga host instance.
config_file (str, optional): the Icinga configuration file to check for the command file directive.
Returns:
str: the Icinga command file path on the Icinga host.
Raises:
spicerack.icinga.IcingaError: if unable to get the command file path.
"""
# Can't use functools cache decorators because NodeSet are not hashable
if len(icinga_host) != 1:
raise IcingaError(
f"Icinga host must match a single host, got: {icinga_host}")
identifier = (str(icinga_host), config_file)
if identifier in cls._command_files:
return cast(CommandFile, cls._command_files[identifier])
try:
# Get the command_file value in the Icinga configuration.
command = r"grep -P '\s*command_file\s*=.+' " + config_file
command_file = ""
for _, output in icinga_host.run_sync(
command,
is_safe=True,
print_output=False,
print_progress_bars=False): # Read only operation
command_file = output.message().decode().split("=",
1)[-1].strip()
if not command_file:
raise ValueError(
f"Empty or no value found for command_file configuration in {config_file}"
)
except (SpicerackError, ValueError) as e:
raise IcingaError(
f"Unable to read command_file configuration in {config_file}"
) from e
cls._command_files[identifier] = command_file
return cast(CommandFile, command_file)
def test_passes_correct_key_file(self):
"""Test that passes correct key file by default."""
expected_key_file = "/etc/etcd/ssl/test0.local.host.priv"
mock_run_sync = _get_mock_run_sync(return_value=b"")
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
controller.get_cluster_info()
_assert_called_with_single_param(
param=f"--key-file {expected_key_file}",
mock_obj=mock_run_sync,
)
def test_passes_correct_endpoints(self):
"""Test that passes correct endpoints by default."""
expected_endpoints = "https://test0.local.host:2379"
mock_run_sync = _get_mock_run_sync(return_value=b"")
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
controller.get_cluster_info()
_assert_called_with_single_param(
param=f"--endpoints {expected_endpoints}",
mock_obj=mock_run_sync,
)
def test_raises_when_no_global_cluster_health(self):
"""Test that parses cluster global status when unhealthy."""
mock_run_sync = _get_mock_run_sync(
return_value=b"""
member 5208bbf5c00e7cdf is unhealthy: got unhealthy result from https://toolsbeta-test-k8s-etcd-6.toolsbeta.eqiad1.wikimedia.cloud:2379
""", # noqa: E501
)
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
with self.assertRaises(UnableToParseOutput):
controller.get_cluster_health()
mock_run_sync.assert_called_once()
def test_raises_when_getting_member_without_id(self):
"""Test that raises when getting member without id."""
mock_run_sync = _get_mock_run_sync(return_value=b"""
415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true
peerURLs=https://idontexist.localhost:1234
"""
# noqa: E501
)
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with self.assertRaises(UnableToParseOutput):
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
controller.get_cluster_info()
def test_parses_result_with_one_member(self):
"""Test that parses result with one member."""
expected_members = {"415090d15def9053": HealthStatus.healthy}
mock_run_sync = _get_mock_run_sync(
return_value=b"""
member 415090d15def9053 is healthy: got healthy result from https://toolsbeta-test-k8s-etcd-6.toolsbeta.eqiad1.wikimedia.cloud:2379
cluster is healthy
""", # noqa: E501
)
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
gotten_result = controller.get_cluster_health()
mock_run_sync.assert_called_once()
assert gotten_result.members_status == expected_members
def test_gets_cluster_unhealthy(self):
"""Test that parses cluster global status when unhealthy."""
expected_global_status = HealthStatus.unhealthy
mock_run_sync = _get_mock_run_sync(
return_value=b"""
member 5208bbf5c00e7cdf is unhealthy: got unhealthy result from https://toolsbeta-test-k8s-etcd-6.toolsbeta.eqiad1.wikimedia.cloud:2379
cluster is unhealthy
""", # noqa: E501
)
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
gotten_result = controller.get_cluster_health()
mock_run_sync.assert_called_once()
assert expected_global_status == gotten_result.global_status
def test_passes_correct_ca_file(self):
"""Test that passes correct ca file by default."""
expected_ca_file = "/etc/etcd/ssl/ca.pem"
mock_run_sync = _get_mock_run_sync(
return_value=b"""
member 415090d15def9053 is healthy: got healthy result from https://toolsbeta-test-k8s-etcd-6.toolsbeta.eqiad1.wikimedia.cloud:2379
cluster is healthy
""", # noqa: E501
)
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
controller.get_cluster_health()
_assert_called_with_single_param(
param=f"--ca-file {expected_ca_file}",
mock_obj=mock_run_sync,
)
def test_skips_removal_if_member_does_not_exist(self):
"""Test that skips removal if member does not exist."""
non_existing_member_fqdn = "i.dont.exist"
expected_result = None
mock_run_sync = _get_mock_run_sync(return_value="""
415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true
""".encode() # noqa: E501
)
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
gotten_result = controller.ensure_node_does_not_exist(
member_fqdn=non_existing_member_fqdn, )
_assert_called_with_single_param(param="list", mock_obj=mock_run_sync)
_assert_not_called_with_single_param(param="remove",
mock_obj=mock_run_sync)
assert gotten_result == expected_result
def test_uses_default_member_url_if_not_passed(self):
"""Test that uses default member url if not passed."""
new_member_fqdn = "i.already.exist"
expected_peer_url = f"https://{new_member_fqdn}:2380"
mock_run_sync = _get_mock_run_sync(side_effect=[
b"",
b"""Added :)""",
f"""
415090d15def9053: name={new_member_fqdn} peerURLs={expected_peer_url}
""".encode(),
])
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
controller.ensure_node_exists(new_member_fqdn=new_member_fqdn)
_assert_called_with_single_param(param=expected_peer_url,
mock_obj=mock_run_sync)
def test_parses_result_with_member_down(self):
"""Test that parses result with member down."""
expected_result = {
"415090d15def9053": {
"member_id":
"415090d15def9053",
"name":
"toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud",
"status":
"up",
"isLeader":
True,
"peerURLs":
"https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380",
"clientURLs":
"https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379",
},
"cf612c785df58f6a": {
"member_id": "cf612c785df58f6a",
"peerURLs": "https://idontexist.localhost:1234",
"status": "unstarted",
},
}
mock_run_sync = _get_mock_run_sync(return_value=b"""
415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true
cf612c785df58f6a[unstarted]: peerURLs=https://idontexist.localhost:1234
"""
# noqa: E501
)
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
gotten_result = controller.get_cluster_info()
mock_run_sync.assert_called_once()
assert expected_result == gotten_result
def test_removes_the_member_if_there_already(self):
"""Test that it removes the member if there already."""
member_fqdn = "i.already.exist"
expected_member_id = "1234556789012345"
mock_run_sync = _get_mock_run_sync(side_effect=[
f"""
415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=false
{expected_member_id}: name={member_fqdn} peerURLs=http://some.url
""".encode(), # noqa: E501
"Removed :)",
])
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
gotten_result = controller.ensure_node_does_not_exist(
member_fqdn=member_fqdn)
_assert_called_with_single_param(param="remove",
mock_obj=mock_run_sync)
assert gotten_result == expected_member_id
def remove_configuration(self, configuration: DHCPConfiguration, force: bool = False) -> None:
"""Remove configuration from target DHCP server then call refresh_dhcp.
This will fail if contents do not match unless force is True.
Arguments:
configuration (spicerack.dhcp.DHCPConfiguration): An instance which provides content and filename for a
configuration.
force (bool, default False): If set to True, will remove filename regardless.
"""
if not force:
confsha256 = sha256(str(configuration).encode()).hexdigest()
try:
results = self._hosts.run_sync(
f"sha256sum {DHCP_TARGET_PATH}/{configuration.filename}",
is_safe=True,
print_output=False,
print_progress_bars=False,
)
except RemoteExecutionError as exc:
raise DHCPError(f"Can't test {configuration.filename} for removal.") from exc
seen_match = False
for _, result in RemoteHosts.results_to_list(results):
remotesha256 = result.strip().split()[0]
if remotesha256 != confsha256 and not self._dry_run:
raise DHCPError(f"Remote {configuration.filename} has a mismatched SHA256, refusing to remove.")
seen_match = True
if not seen_match:
raise DHCPError("Did not get any result trying to get SHA256, refusing to attempt to remove.")
try:
self._hosts.run_sync(
f"/bin/rm -v {DHCP_TARGET_PATH}/{configuration.filename}", print_output=False, print_progress_bars=False
)
except RemoteExecutionError as exc:
raise DHCPError(f"Can't remove {configuration.filename}.") from exc
self.refresh_dhcp()
def test_skips_addition_if_member_already_exists(self):
"""Test that skips addition if member already exists."""
existing_member_fqdn = "i.already.exist"
existing_member_peer_url = f"https://{existing_member_fqdn}:1234"
expected_member_id = "1234556789012345"
mock_run_sync = _get_mock_run_sync(return_value=f"""
415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true
{expected_member_id}: name={existing_member_fqdn} peerURLs={existing_member_peer_url}
""".encode() # noqa: E501
)
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
gotten_member_id = controller.ensure_node_exists(
new_member_fqdn=existing_member_fqdn,
member_peer_url=existing_member_peer_url,
)
_assert_called_with_single_param(param="list", mock_obj=mock_run_sync)
_assert_not_called_with_single_param(param="add",
mock_obj=mock_run_sync)
assert gotten_member_id == expected_member_id
def test_get_core_dbs_ok(self, kwargs, query, match):
"""It should return the right DBs based on the parameters."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet(match))
self.mysql.get_core_dbs(**kwargs)
self.mocked_remote.query.assert_called_once_with(query)
def test_set_core_masters_readonly(self, mode, value, caplog):
"""It should set the masters as read-only/read-write."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db10[01-11]"))
mock_cumin(self.mocked_transports, 0, retvals=[[("db10[01-11]", value)]])
getattr(self.mysql, "set_core_masters_" + mode)("eqiad")
assert "SET GLOBAL read_only=" + value.decode() in caplog.text
def test_verify_core_masters_readonly_ok(self, readonly, reply, caplog):
"""Should verify that the masters have the intended read-only value."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db10[01-11]"))
mock_cumin(self.mocked_transports, 0, retvals=[[("db10[01-11]", reply)]])
self.mysql.verify_core_masters_readonly("eqiad", readonly)
assert "SELECT @@global.read_only" in caplog.text