def test_sign_fail(self):
"""It should raise PuppetMasterError if the sign operation fails."""
requested_results = [(
NodeSet("puppetmaster.example.com"),
MsgTreeElem(PUPPET_CA_CERT_METADATA_REQUESTED,
parent=MsgTreeElem()),
)]
sign_results = [(
NodeSet("puppetmaster.example.com"),
MsgTreeElem(b"sign error", parent=MsgTreeElem()),
)]
signed_results = [(
NodeSet("puppetmaster.example.com"),
MsgTreeElem(PUPPET_CA_CERT_METADATA_REQUESTED,
parent=MsgTreeElem()),
)]
self.mocked_master_host.run_sync.side_effect = [
iter(requested_results),
iter(sign_results),
iter(signed_results),
]
with pytest.raises(
puppet.PuppetMasterError,
match=
"Expected certificate for test.example.com to be signed, got: requested",
):
self.puppet_master.sign("test.example.com", "00:AA")
def _reboot(self, hosts: NodeSet) -> None:
"""Reboot a set of hosts with downtime
Arguments:
hosts (`NodeSet`): A list of hosts to reboot
"""
puppet = self._spicerack.puppet(hosts)
icinga_hosts = self._spicerack.icinga_hosts(hosts.hosts)
try:
duration = timedelta(minutes=20)
with icinga_hosts.downtimed(self.reason, duration=duration):
reboot_time = datetime.utcnow()
confirm_on_failure(hosts.reboot, batch_size=len(hosts))
hosts.wait_reboot_since(reboot_time, print_progress_bars=False)
puppet.run(quiet=True)
puppet.wait_since(reboot_time)
icinga_hosts.wait_for_optimal()
self.results.success(hosts.hosts)
except IcingaError as error:
ask_confirmation(f'Failed to downtime hosts: {error}')
self.logger.warning(error)
except AbortError as error:
# Some host failed to come up again, or something fundamental broke.
# log an error, continue *without* repooling
self.logger.error(error)
self.logger.error(
'Error rebooting: Hosts %s, they may still be depooled', hosts)
self.results.fail(hosts.hosts)
raise
def test_check_core_masters_in_sync_ok(self, mocked_sleep):
"""Should check that all core masters are in sync with the master in the other DC."""
hosts = NodeSet(EQIAD_CORE_MASTERS_QUERY)
self.mocked_remote.query.side_effect = [RemoteHosts(self.config, NodeSet(host)) for host in hosts] * 2
retvals = [[(host, b"2018-09-06T10:00:00.000000")] for host in hosts] # first heartbeat
retvals += [[(host, b"2018-09-06T10:00:01.000000")] for host in hosts] # second heartbeat
mock_cumin(self.mocked_transports, 0, retvals=retvals)
self.mysql.check_core_masters_in_sync("eqiad", "codfw")
assert not mocked_sleep.called
def setup_method(self, _, mocked_transports):
"""Setup the test environment."""
# pylint: disable=attribute-defined-outside-init
self.config = Config(get_fixture_path("remote", "config.yaml"))
self.mocked_transports = mocked_transports
self.mysql_remote_hosts = mysql_legacy.MysqlLegacyRemoteHosts(
RemoteHosts(self.config, NodeSet("host[1-9]"), dry_run=False)
)
self.expected = [(NodeSet("host1"), "output1")]
def test_check_disabled_ok(self):
"""It should check that all hosts have Puppet disabled."""
host1 = NodeSet("test1.example.com")
host2 = NodeSet("test2.example.com")
results = [
(host1, MsgTreeElem(b"1", parent=MsgTreeElem())),
(host2, MsgTreeElem(b"1", parent=MsgTreeElem())),
]
self.mocked_remote_hosts.run_sync.return_value = iter(results)
self.puppet_hosts.check_disabled()
def test_uptime_ok(self):
"""It should gather the current uptime from the target hosts."""
nodes_a = "host1"
nodes_b = "host[2-9]"
mock_cumin(
self.mocked_transports,
0,
retvals=[[(nodes_a, b"1514768400"), (nodes_b, b"1514768401")]],
)
uptimes = self.remote_hosts.uptime()
assert sorted(uptimes) == sorted([(NodeSet(nodes_a), 1514768400.0), (NodeSet(nodes_b), 1514768401.0)])
def main():
"""Check Cumin aliases for inconsistencies.
Note:
Those are the performed checks
- each alias should return some hosts.
- the sum of all DC-related aliases should return all hosts.
- the sum of all the other aliases should return all hosts.
Returns:
int: zero on success, positive integer on failure.
"""
ret = 0
config = Config()
dc_hosts = NodeSet()
alias_hosts = NodeSet()
all_hosts = query.Query(config).execute('*')
for alias in config['aliases']:
try:
match = query.Query(config).execute('A:' + alias)
except InvalidQueryError as e:
print('Unable to execute query for alias {alias}: {msg}'.format(
alias=alias, msg=e))
ret = 1
continue
if not match:
print('Alias {alias} matched 0 hosts'.format(alias=alias))
ret = 1
if alias in DCS:
dc_hosts |= match
else:
alias_hosts |= match
time.sleep(2) # Go gentle on PuppetDB
base_ret = 2
for hosts, name in ((dc_hosts, 'DC'), (alias_hosts, 'Other')):
if all_hosts - hosts:
print('{name} aliases do not cover all hosts: {hosts}'.format(
name=name, hosts=(all_hosts - hosts)))
ret += base_ret
elif dc_hosts - all_hosts:
print('{name} aliases have unknown hosts: {hosts}'.format(
name=name, hosts=(hosts - all_hosts)))
ret += base_ret * 2
base_ret *= 4
return ret
def test_check_enabled_raise(self):
"""It should raise PuppetHostsCheckError if Puppet is disabled on some hosts."""
host1 = NodeSet("test1.example.com")
host2 = NodeSet("test2.example.com")
results = [
(host1, MsgTreeElem(b"0", parent=MsgTreeElem())),
(host2, MsgTreeElem(b"1", parent=MsgTreeElem())),
]
self.mocked_remote_hosts.run_sync.return_value = iter(results)
with pytest.raises(
puppet.PuppetHostsCheckError,
match="Puppet is not enabled on those hosts: test2.example.com",
):
self.puppet_hosts.check_enabled()
def test_check_core_masters_in_sync_not_in_sync(self, mocked_sleep):
"""Should raise MysqlLegacyError if a master is not in sync with the one in the other DC."""
hosts = NodeSet(EQIAD_CORE_MASTERS_QUERY)
self.mocked_remote.query.side_effect = [RemoteHosts(self.config, NodeSet(host)) for host in hosts] + [
RemoteHosts(self.config, NodeSet("db1001"))
] * 3
retvals = [[(host, b"2018-09-06T10:00:00.000000")] for host in hosts] # first heartbeat
retvals += [[("db1001", b"2018-09-06T10:00:00.000000")]] * 3 # 3 failed retries of second heartbeat
mock_cumin(self.mocked_transports, 0, retvals=retvals)
with pytest.raises(
mysql_legacy.MysqlLegacyError,
match=r"Heartbeat from master db1001 for section .* not yet in sync",
):
self.mysql.check_core_masters_in_sync("eqiad", "codfw")
assert mocked_sleep.called
def test_check_core_masters_in_sync_fail_heartbeat(self, mocked_sleep):
"""Should raise MysqlLegacyError if unable to get the heartbeat from the current master."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001"))
mock_cumin(self.mocked_transports, 0, retvals=[])
with pytest.raises(mysql_legacy.MysqlLegacyError, match="Unable to get heartbeat from master"):
self.mysql.check_core_masters_in_sync("eqiad", "codfw")
assert not mocked_sleep.called
def __init__(self,
icinga_host: RemoteHosts,
target_hosts: TypeHosts,
*,
verbatim_hosts: bool = False) -> None:
"""Initialize the instance.
Arguments:
icinga_host (spicerack.remote.RemoteHosts): the RemoteHosts instance for the Icinga server.
target_hosts (spicerack.typing.TypeHosts): the target hosts either as a NodeSet instance or a sequence of
strings.
verbatim_hosts (bool, optional): if :py:data:`True` use the hosts passed verbatim as is, if instead
:py:data:`False`, the default, consider the given target hosts as FQDNs and extract their hostnames to
be used in Icinga.
"""
if not verbatim_hosts:
target_hosts = [
target_host.split(".")[0] for target_host in target_hosts
]
if isinstance(target_hosts, NodeSet):
self._target_hosts = target_hosts
else:
self._target_hosts = NodeSet.fromlist(target_hosts)
if not self._target_hosts:
raise IcingaError("Got empty target hosts list.")
self._command_file = CommandFile(
icinga_host
) # This validates also that icinga_host matches a single server.
self._icinga_host = icinga_host
self._verbatim_hosts = verbatim_hosts
def test_adds_the_member_if_not_there(self):
"""Test that adds the member if not there."""
new_member_fqdn = "i.already.exist"
new_member_peer_url = f"https://{new_member_fqdn}:1234"
expected_member_id = "1234556789012345"
mock_run_sync = _get_mock_run_sync(side_effect=[
b"""
415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true
""", # noqa: E501
b"""Added :)""",
f"""
415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=false
{expected_member_id}: name={new_member_fqdn} peerURLs={new_member_peer_url}
""".encode(), # noqa: E501
])
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
gotten_member_id = controller.ensure_node_exists(
new_member_fqdn=new_member_fqdn,
member_peer_url=new_member_peer_url,
)
_assert_called_with_single_param(param="add", mock_obj=mock_run_sync)
assert gotten_member_id == expected_member_id
def test_raises_if_more_than_one_node_is_used(self):
"""Test that raises if more than one node is used."""
nodes = RemoteHosts(config=mock.MagicMock(specset=Config),
hosts=NodeSet("test[0,1].local.host"))
with self.assertRaises(TooManyHosts):
EtcdctlController(remote_host=nodes)
def test_parses_result_with_one_member(self):
"""Test that parses result with one member."""
expected_node = {
"member_id":
"415090d15def9053",
"name":
"toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud",
"status":
"up",
"isLeader":
True,
"peerURLs":
"https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380",
"clientURLs":
"https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379",
}
mock_run_sync = _get_mock_run_sync(
return_value=b"""
415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true
""", # noqa: E501
)
controller = EtcdctlController(remote_host=RemoteHosts(
config=mock.MagicMock(specset=Config),
hosts=NodeSet("test0.local.host")), )
with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync):
gotten_result = controller.get_cluster_info()
mock_run_sync.assert_called_once()
assert len(gotten_result) == 1
assert expected_node["member_id"] in gotten_result
assert gotten_result[expected_node["member_id"]] == expected_node
def test_get_core_dbs_fail_sanity_check(self):
"""It should raise MysqlLegacyError if matching an invalid number of hosts when looking for masters."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001"))
with pytest.raises(mysql_legacy.MysqlLegacyError, match="Matched 1 masters, expected 12"):
self.mysql.get_core_dbs(datacenter="eqiad", replication_role="master")
assert self.mocked_remote.query.called
def test_get_remote_hosts():
"""Test that RemoteHosts instance is returned."""
mocked_remote_hosts = mock.Mock(spec_set=RemoteHosts)
mocked_remote_hosts.hosts = NodeSet("el[1-2]")
elastic_hosts = ec.ElasticsearchHosts(mocked_remote_hosts, None)
result = elastic_hosts.get_remote_hosts()
assert isinstance(result, RemoteHosts)
def mock_cumin(mocked_transports, retcode, retvals=None):
"""Given a mocked cumin.transports, add the necessary mocks for these tests and set the retcode."""
if retvals is None:
retvals = [[("host1", b"output1")]]
results = []
for retval in retvals:
result = []
for host, message in retval:
result.append((NodeSet(host), MsgTreeElem(message, parent=MsgTreeElem())))
results.append(result)
mocked_transports.clustershell = clustershell
mocked_execute = mock.Mock()
mocked_execute.return_value = retcode
mocked_get_results = mock.Mock()
if results:
mocked_get_results.side_effect = results
else:
mocked_get_results.return_value = iter(())
mocked_transports.clustershell.ClusterShellWorker.execute = mocked_execute
mocked_transports.clustershell.ClusterShellWorker.get_results = mocked_get_results
mocked_transports.Target = Target
def test_reboot_single(self, mocked_target):
"""It should call the reboot script on the target host with default batch size and no sleep."""
hosts = NodeSet("host1")
remote_hosts = remote.RemoteHosts(self.config, hosts, dry_run=False)
mock_cumin(self.mocked_transports, 0)
remote_hosts.reboot()
self.mocked_transports.clustershell.ClusterShellWorker.execute.assert_called_once_with()
mocked_target.assert_has_calls([mock.call(hosts, batch_size_ratio=None, batch_sleep=None, batch_size=1)])
def set_mocked_icinga_host_outputs(mocked_icinga_host, outputs):
"""Setup the mocked icinga_host side effect to return the given outputs, one after another."""
outs = [
MsgTreeElem(output.encode(), parent=MsgTreeElem())
for output in outputs
]
mocked_icinga_host.run_sync.side_effect = [
iter([(NodeSet("icinga-host"), out)]) for out in outs
]
def test_query_ok(self):
"""Calling query() should return the matching hosts."""
query = "host[1-9]"
remote_hosts = self.remote.query(query)
assert isinstance(remote_hosts, remote.RemoteHosts)
assert remote_hosts.hosts == NodeSet(query)
assert str(remote_hosts) == "host[1-9]"
assert len(remote_hosts) == 9
def test_query_accepts_sudo(self):
"""Calling query() should return the matching hosts even if using sudo."""
query = "host[1-9]"
remote_hosts = self.remote.query(query, use_sudo=True)
assert isinstance(remote_hosts, remote.RemoteHosts)
assert remote_hosts.hosts == NodeSet(query)
assert str(remote_hosts) == "host[1-9]"
assert len(remote_hosts) == 9
assert remote_hosts._use_sudo # pylint: disable=protected-access
def setup_method(self):
"""Setup the test environment."""
# pylint: disable=attribute-defined-outside-init
config = get_fixture_path("remote", "config.yaml")
self.hosts = NodeSet("host[1-10]")
# We want to mock out ConftoolEntity completely here. As far as we're concerned it's just an interface
self.conftool = mock.MagicMock(spec=confctl.ConftoolEntity)
self.remote_hosts = remote.RemoteHosts(config, self.hosts, dry_run=False)
self.remote_hosts.run_async = mock.MagicMock()
self.lbcluster = remote.LBRemoteCluster(config, self.remote_hosts, self.conftool)
def test_get_core_masters_heartbeats_wrong_data(self):
"""Should raise MysqlLegacyError if unable to convert the heartbeat into a datetime."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001"))
mock_cumin(
self.mocked_transports,
0,
retvals=[[("db1001", b"2018-09-06-10:00:00.000000")]],
)
with pytest.raises(mysql_legacy.MysqlLegacyError, match="Unable to convert heartbeat"):
self.mysql.get_core_masters_heartbeats("eqiad", "codfw")
def main():
"""Check Cumin aliases for inconsistencies.
Note:
Those are the performed checks
- each alias should return some hosts, unless listed in OPTIONAL_ALIASES.
- the sum of all DC-related aliases should return all hosts.
- the sum of all the other aliases should return all hosts.
"""
config = Config()
dc_hosts = NodeSet()
alias_hosts = NodeSet()
all_hosts = query.Query(config).execute('*')
for alias in config['aliases']:
try:
match = query.Query(config).execute('A:' + alias)
except InvalidQueryError as e:
print('Unable to execute query for alias {alias}: {msg}'.format(
alias=alias, msg=e))
continue
if not match and alias not in OPTIONAL_ALIASES:
print('Alias {alias} matched 0 hosts'.format(alias=alias))
if alias in DCS:
dc_hosts |= match
else:
alias_hosts |= match
time.sleep(2) # Go gentle on PuppetDB
for hosts, name in ((dc_hosts, 'DC'), (alias_hosts, 'Other')):
if all_hosts - hosts:
print('{name} aliases do not cover all hosts: {hosts}'.format(
name=name, hosts=(all_hosts - hosts)))
elif dc_hosts - all_hosts:
print('{name} aliases have unknown hosts: {hosts}'.format(
name=name, hosts=(hosts - all_hosts)))
return 0
def test_sign_alt_dns(self):
"""It should pass the --allow-dns-alt-names option while signing the certificate."""
requested_results = [(
NodeSet("puppetmaster.example.com"),
MsgTreeElem(PUPPET_CA_CERT_METADATA_REQUESTED,
parent=MsgTreeElem()),
)]
signed_results = [(
NodeSet("puppetmaster.example.com"),
MsgTreeElem(PUPPET_CA_CERT_METADATA_SIGNED, parent=MsgTreeElem()),
)]
self.mocked_master_host.run_sync.side_effect = [
iter(requested_results),
iter(()),
iter(signed_results),
]
self.puppet_master.sign("test.example.com",
"00:AA",
allow_alt_names=True)
self.mocked_master_host.run_sync.assert_has_calls([
mock.call(
"puppet ca --disable_warnings deprecations --render-as json "
r'list --all --subject "^test\.example\.com$"',
is_safe=True,
print_output=False,
print_progress_bars=False,
),
mock.call(
"puppet cert --disable_warnings deprecations sign --allow-dns-alt-names test.example.com",
print_output=False,
print_progress_bars=False,
),
mock.call(
"puppet ca --disable_warnings deprecations --render-as json "
r'list --all --subject "^test\.example\.com$"',
is_safe=True,
print_output=False,
print_progress_bars=False,
),
])
def test_using_sudo_prepends_when_command_is_string(self, mocked_transport_new):
"""Test that using sudo prepends when command is string."""
nodes_a = "host1"
mock_cumin(self.mocked_transports, 0, retvals=[[(nodes_a, b"")]])
mocked_worker = mock.MagicMock()
mocked_worker.execute.return_value = 0
mocked_transport_new.return_value = mocked_worker
remote.RemoteHosts(self.config, NodeSet(nodes_a), dry_run=False, use_sudo=True).run_sync("command")
assert mocked_worker.commands == ["sudo -i command"]
def test_wait_since_failed_execution(self, mocked_sleep):
"""It should raise PuppetHostsCheckError if fails to get the successful Puppet run within the timeout."""
self.mocked_remote_hosts.run_sync.side_effect = RemoteExecutionError(
1, "fail")
self.mocked_remote_hosts.hosts = NodeSet("test.example.com")
with pytest.raises(puppet.PuppetHostsCheckError,
match="Unable to find a successful Puppet run"):
self.puppet_hosts.wait_since(datetime.utcnow())
assert mocked_sleep.called
def test_get_certificate_metadata_raises(self, json_output,
exception_message):
"""It should raise PuppetMasterError if the Puppet CA returns multiple certificates metadata."""
results = [(
NodeSet("puppetmaster.example.com"),
MsgTreeElem(json_output, parent=MsgTreeElem()),
)]
self.mocked_master_host.run_sync.return_value = iter(results)
with pytest.raises(puppet.PuppetMasterError, match=exception_message):
self.puppet_master.get_certificate_metadata("test.example.com")
def test_get_ca_servers_handles_multiple_results(self):
"""Test test get ca servers handles multiple results."""
self.mocked_remote_hosts.run_sync.return_value = [
(
NodeSet("test0.example.com"),
MsgTreeElem(b"test0.puppetmast.er", parent=MsgTreeElem()),
),
(
NodeSet("test1.example.com"),
MsgTreeElem(b"test1.puppetmast.er", parent=MsgTreeElem()),
),
]
result = self.puppet_hosts.get_ca_servers()
self.mocked_remote_hosts.run_sync.assert_called_once()
assert "test0.example.com" in result
assert result["test0.example.com"] == "test0.puppetmast.er"
assert "test1.example.com" in result
assert result["test1.example.com"] == "test1.puppetmast.er"
def _get_disabled(self) -> Dict[bool, NodeSet]:
"""Check if Puppet is disabled on the hosts.
Returns:
dict: a dict with :py:class:`bool` keys for Puppet disabled or not and hosts
:py:class:`ClusterShell.NodeSet.NodeSet` as values.
"""
results = self._remote_hosts.run_sync(
f'source {PUPPET_COMMON_SCRIPT} && test -f "${{PUPPET_DISABLEDLOCK}}" && echo "1" || echo "0"',
is_safe=True,
print_output=False,
print_progress_bars=False,
)
disabled = {True: NodeSet(), False: NodeSet()}
for nodeset, output in results:
result = bool(int(output.message().decode().strip()))
disabled[result] |= nodeset
return disabled
