def test_sign_fail(self): """It should raise PuppetMasterError if the sign operation fails.""" requested_results = [( NodeSet("puppetmaster.example.com"), MsgTreeElem(PUPPET_CA_CERT_METADATA_REQUESTED, parent=MsgTreeElem()), )] sign_results = [( NodeSet("puppetmaster.example.com"), MsgTreeElem(b"sign error", parent=MsgTreeElem()), )] signed_results = [( NodeSet("puppetmaster.example.com"), MsgTreeElem(PUPPET_CA_CERT_METADATA_REQUESTED, parent=MsgTreeElem()), )] self.mocked_master_host.run_sync.side_effect = [ iter(requested_results), iter(sign_results), iter(signed_results), ] with pytest.raises( puppet.PuppetMasterError, match= "Expected certificate for test.example.com to be signed, got: requested", ): self.puppet_master.sign("test.example.com", "00:AA")
def _reboot(self, hosts: NodeSet) -> None: """Reboot a set of hosts with downtime Arguments: hosts (`NodeSet`): A list of hosts to reboot """ puppet = self._spicerack.puppet(hosts) icinga_hosts = self._spicerack.icinga_hosts(hosts.hosts) try: duration = timedelta(minutes=20) with icinga_hosts.downtimed(self.reason, duration=duration): reboot_time = datetime.utcnow() confirm_on_failure(hosts.reboot, batch_size=len(hosts)) hosts.wait_reboot_since(reboot_time, print_progress_bars=False) puppet.run(quiet=True) puppet.wait_since(reboot_time) icinga_hosts.wait_for_optimal() self.results.success(hosts.hosts) except IcingaError as error: ask_confirmation(f'Failed to downtime hosts: {error}') self.logger.warning(error) except AbortError as error: # Some host failed to come up again, or something fundamental broke. # log an error, continue *without* repooling self.logger.error(error) self.logger.error( 'Error rebooting: Hosts %s, they may still be depooled', hosts) self.results.fail(hosts.hosts) raise
def test_check_core_masters_in_sync_ok(self, mocked_sleep): """Should check that all core masters are in sync with the master in the other DC.""" hosts = NodeSet(EQIAD_CORE_MASTERS_QUERY) self.mocked_remote.query.side_effect = [RemoteHosts(self.config, NodeSet(host)) for host in hosts] * 2 retvals = [[(host, b"2018-09-06T10:00:00.000000")] for host in hosts] # first heartbeat retvals += [[(host, b"2018-09-06T10:00:01.000000")] for host in hosts] # second heartbeat mock_cumin(self.mocked_transports, 0, retvals=retvals) self.mysql.check_core_masters_in_sync("eqiad", "codfw") assert not mocked_sleep.called
def setup_method(self, _, mocked_transports): """Setup the test environment.""" # pylint: disable=attribute-defined-outside-init self.config = Config(get_fixture_path("remote", "config.yaml")) self.mocked_transports = mocked_transports self.mysql_remote_hosts = mysql_legacy.MysqlLegacyRemoteHosts( RemoteHosts(self.config, NodeSet("host[1-9]"), dry_run=False) ) self.expected = [(NodeSet("host1"), "output1")]
def test_check_disabled_ok(self): """It should check that all hosts have Puppet disabled.""" host1 = NodeSet("test1.example.com") host2 = NodeSet("test2.example.com") results = [ (host1, MsgTreeElem(b"1", parent=MsgTreeElem())), (host2, MsgTreeElem(b"1", parent=MsgTreeElem())), ] self.mocked_remote_hosts.run_sync.return_value = iter(results) self.puppet_hosts.check_disabled()
def test_uptime_ok(self): """It should gather the current uptime from the target hosts.""" nodes_a = "host1" nodes_b = "host[2-9]" mock_cumin( self.mocked_transports, 0, retvals=[[(nodes_a, b"1514768400"), (nodes_b, b"1514768401")]], ) uptimes = self.remote_hosts.uptime() assert sorted(uptimes) == sorted([(NodeSet(nodes_a), 1514768400.0), (NodeSet(nodes_b), 1514768401.0)])
def main(): """Check Cumin aliases for inconsistencies. Note: Those are the performed checks - each alias should return some hosts. - the sum of all DC-related aliases should return all hosts. - the sum of all the other aliases should return all hosts. Returns: int: zero on success, positive integer on failure. """ ret = 0 config = Config() dc_hosts = NodeSet() alias_hosts = NodeSet() all_hosts = query.Query(config).execute('*') for alias in config['aliases']: try: match = query.Query(config).execute('A:' + alias) except InvalidQueryError as e: print('Unable to execute query for alias {alias}: {msg}'.format( alias=alias, msg=e)) ret = 1 continue if not match: print('Alias {alias} matched 0 hosts'.format(alias=alias)) ret = 1 if alias in DCS: dc_hosts |= match else: alias_hosts |= match time.sleep(2) # Go gentle on PuppetDB base_ret = 2 for hosts, name in ((dc_hosts, 'DC'), (alias_hosts, 'Other')): if all_hosts - hosts: print('{name} aliases do not cover all hosts: {hosts}'.format( name=name, hosts=(all_hosts - hosts))) ret += base_ret elif dc_hosts - all_hosts: print('{name} aliases have unknown hosts: {hosts}'.format( name=name, hosts=(hosts - all_hosts))) ret += base_ret * 2 base_ret *= 4 return ret
def test_check_enabled_raise(self): """It should raise PuppetHostsCheckError if Puppet is disabled on some hosts.""" host1 = NodeSet("test1.example.com") host2 = NodeSet("test2.example.com") results = [ (host1, MsgTreeElem(b"0", parent=MsgTreeElem())), (host2, MsgTreeElem(b"1", parent=MsgTreeElem())), ] self.mocked_remote_hosts.run_sync.return_value = iter(results) with pytest.raises( puppet.PuppetHostsCheckError, match="Puppet is not enabled on those hosts: test2.example.com", ): self.puppet_hosts.check_enabled()
def test_check_core_masters_in_sync_not_in_sync(self, mocked_sleep): """Should raise MysqlLegacyError if a master is not in sync with the one in the other DC.""" hosts = NodeSet(EQIAD_CORE_MASTERS_QUERY) self.mocked_remote.query.side_effect = [RemoteHosts(self.config, NodeSet(host)) for host in hosts] + [ RemoteHosts(self.config, NodeSet("db1001")) ] * 3 retvals = [[(host, b"2018-09-06T10:00:00.000000")] for host in hosts] # first heartbeat retvals += [[("db1001", b"2018-09-06T10:00:00.000000")]] * 3 # 3 failed retries of second heartbeat mock_cumin(self.mocked_transports, 0, retvals=retvals) with pytest.raises( mysql_legacy.MysqlLegacyError, match=r"Heartbeat from master db1001 for section .* not yet in sync", ): self.mysql.check_core_masters_in_sync("eqiad", "codfw") assert mocked_sleep.called
def test_check_core_masters_in_sync_fail_heartbeat(self, mocked_sleep): """Should raise MysqlLegacyError if unable to get the heartbeat from the current master.""" self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001")) mock_cumin(self.mocked_transports, 0, retvals=[]) with pytest.raises(mysql_legacy.MysqlLegacyError, match="Unable to get heartbeat from master"): self.mysql.check_core_masters_in_sync("eqiad", "codfw") assert not mocked_sleep.called
def __init__(self, icinga_host: RemoteHosts, target_hosts: TypeHosts, *, verbatim_hosts: bool = False) -> None: """Initialize the instance. Arguments: icinga_host (spicerack.remote.RemoteHosts): the RemoteHosts instance for the Icinga server. target_hosts (spicerack.typing.TypeHosts): the target hosts either as a NodeSet instance or a sequence of strings. verbatim_hosts (bool, optional): if :py:data:`True` use the hosts passed verbatim as is, if instead :py:data:`False`, the default, consider the given target hosts as FQDNs and extract their hostnames to be used in Icinga. """ if not verbatim_hosts: target_hosts = [ target_host.split(".")[0] for target_host in target_hosts ] if isinstance(target_hosts, NodeSet): self._target_hosts = target_hosts else: self._target_hosts = NodeSet.fromlist(target_hosts) if not self._target_hosts: raise IcingaError("Got empty target hosts list.") self._command_file = CommandFile( icinga_host ) # This validates also that icinga_host matches a single server. self._icinga_host = icinga_host self._verbatim_hosts = verbatim_hosts
def test_adds_the_member_if_not_there(self): """Test that adds the member if not there.""" new_member_fqdn = "i.already.exist" new_member_peer_url = f"https://{new_member_fqdn}:1234" expected_member_id = "1234556789012345" mock_run_sync = _get_mock_run_sync(side_effect=[ b""" 415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true """, # noqa: E501 b"""Added :)""", f""" 415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=false {expected_member_id}: name={new_member_fqdn} peerURLs={new_member_peer_url} """.encode(), # noqa: E501 ]) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): gotten_member_id = controller.ensure_node_exists( new_member_fqdn=new_member_fqdn, member_peer_url=new_member_peer_url, ) _assert_called_with_single_param(param="add", mock_obj=mock_run_sync) assert gotten_member_id == expected_member_id
def test_raises_if_more_than_one_node_is_used(self): """Test that raises if more than one node is used.""" nodes = RemoteHosts(config=mock.MagicMock(specset=Config), hosts=NodeSet("test[0,1].local.host")) with self.assertRaises(TooManyHosts): EtcdctlController(remote_host=nodes)
def test_parses_result_with_one_member(self): """Test that parses result with one member.""" expected_node = { "member_id": "415090d15def9053", "name": "toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud", "status": "up", "isLeader": True, "peerURLs": "https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380", "clientURLs": "https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379", } mock_run_sync = _get_mock_run_sync( return_value=b""" 415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true """, # noqa: E501 ) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): gotten_result = controller.get_cluster_info() mock_run_sync.assert_called_once() assert len(gotten_result) == 1 assert expected_node["member_id"] in gotten_result assert gotten_result[expected_node["member_id"]] == expected_node
def test_get_core_dbs_fail_sanity_check(self): """It should raise MysqlLegacyError if matching an invalid number of hosts when looking for masters.""" self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001")) with pytest.raises(mysql_legacy.MysqlLegacyError, match="Matched 1 masters, expected 12"): self.mysql.get_core_dbs(datacenter="eqiad", replication_role="master") assert self.mocked_remote.query.called
def test_get_remote_hosts(): """Test that RemoteHosts instance is returned.""" mocked_remote_hosts = mock.Mock(spec_set=RemoteHosts) mocked_remote_hosts.hosts = NodeSet("el[1-2]") elastic_hosts = ec.ElasticsearchHosts(mocked_remote_hosts, None) result = elastic_hosts.get_remote_hosts() assert isinstance(result, RemoteHosts)
def mock_cumin(mocked_transports, retcode, retvals=None): """Given a mocked cumin.transports, add the necessary mocks for these tests and set the retcode.""" if retvals is None: retvals = [[("host1", b"output1")]] results = [] for retval in retvals: result = [] for host, message in retval: result.append((NodeSet(host), MsgTreeElem(message, parent=MsgTreeElem()))) results.append(result) mocked_transports.clustershell = clustershell mocked_execute = mock.Mock() mocked_execute.return_value = retcode mocked_get_results = mock.Mock() if results: mocked_get_results.side_effect = results else: mocked_get_results.return_value = iter(()) mocked_transports.clustershell.ClusterShellWorker.execute = mocked_execute mocked_transports.clustershell.ClusterShellWorker.get_results = mocked_get_results mocked_transports.Target = Target
def test_reboot_single(self, mocked_target): """It should call the reboot script on the target host with default batch size and no sleep.""" hosts = NodeSet("host1") remote_hosts = remote.RemoteHosts(self.config, hosts, dry_run=False) mock_cumin(self.mocked_transports, 0) remote_hosts.reboot() self.mocked_transports.clustershell.ClusterShellWorker.execute.assert_called_once_with() mocked_target.assert_has_calls([mock.call(hosts, batch_size_ratio=None, batch_sleep=None, batch_size=1)])
def set_mocked_icinga_host_outputs(mocked_icinga_host, outputs): """Setup the mocked icinga_host side effect to return the given outputs, one after another.""" outs = [ MsgTreeElem(output.encode(), parent=MsgTreeElem()) for output in outputs ] mocked_icinga_host.run_sync.side_effect = [ iter([(NodeSet("icinga-host"), out)]) for out in outs ]
def test_query_ok(self): """Calling query() should return the matching hosts.""" query = "host[1-9]" remote_hosts = self.remote.query(query) assert isinstance(remote_hosts, remote.RemoteHosts) assert remote_hosts.hosts == NodeSet(query) assert str(remote_hosts) == "host[1-9]" assert len(remote_hosts) == 9
def test_query_accepts_sudo(self): """Calling query() should return the matching hosts even if using sudo.""" query = "host[1-9]" remote_hosts = self.remote.query(query, use_sudo=True) assert isinstance(remote_hosts, remote.RemoteHosts) assert remote_hosts.hosts == NodeSet(query) assert str(remote_hosts) == "host[1-9]" assert len(remote_hosts) == 9 assert remote_hosts._use_sudo # pylint: disable=protected-access
def setup_method(self): """Setup the test environment.""" # pylint: disable=attribute-defined-outside-init config = get_fixture_path("remote", "config.yaml") self.hosts = NodeSet("host[1-10]") # We want to mock out ConftoolEntity completely here. As far as we're concerned it's just an interface self.conftool = mock.MagicMock(spec=confctl.ConftoolEntity) self.remote_hosts = remote.RemoteHosts(config, self.hosts, dry_run=False) self.remote_hosts.run_async = mock.MagicMock() self.lbcluster = remote.LBRemoteCluster(config, self.remote_hosts, self.conftool)
def test_get_core_masters_heartbeats_wrong_data(self): """Should raise MysqlLegacyError if unable to convert the heartbeat into a datetime.""" self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001")) mock_cumin( self.mocked_transports, 0, retvals=[[("db1001", b"2018-09-06-10:00:00.000000")]], ) with pytest.raises(mysql_legacy.MysqlLegacyError, match="Unable to convert heartbeat"): self.mysql.get_core_masters_heartbeats("eqiad", "codfw")
def main(): """Check Cumin aliases for inconsistencies. Note: Those are the performed checks - each alias should return some hosts, unless listed in OPTIONAL_ALIASES. - the sum of all DC-related aliases should return all hosts. - the sum of all the other aliases should return all hosts. """ config = Config() dc_hosts = NodeSet() alias_hosts = NodeSet() all_hosts = query.Query(config).execute('*') for alias in config['aliases']: try: match = query.Query(config).execute('A:' + alias) except InvalidQueryError as e: print('Unable to execute query for alias {alias}: {msg}'.format( alias=alias, msg=e)) continue if not match and alias not in OPTIONAL_ALIASES: print('Alias {alias} matched 0 hosts'.format(alias=alias)) if alias in DCS: dc_hosts |= match else: alias_hosts |= match time.sleep(2) # Go gentle on PuppetDB for hosts, name in ((dc_hosts, 'DC'), (alias_hosts, 'Other')): if all_hosts - hosts: print('{name} aliases do not cover all hosts: {hosts}'.format( name=name, hosts=(all_hosts - hosts))) elif dc_hosts - all_hosts: print('{name} aliases have unknown hosts: {hosts}'.format( name=name, hosts=(hosts - all_hosts))) return 0
def test_sign_alt_dns(self): """It should pass the --allow-dns-alt-names option while signing the certificate.""" requested_results = [( NodeSet("puppetmaster.example.com"), MsgTreeElem(PUPPET_CA_CERT_METADATA_REQUESTED, parent=MsgTreeElem()), )] signed_results = [( NodeSet("puppetmaster.example.com"), MsgTreeElem(PUPPET_CA_CERT_METADATA_SIGNED, parent=MsgTreeElem()), )] self.mocked_master_host.run_sync.side_effect = [ iter(requested_results), iter(()), iter(signed_results), ] self.puppet_master.sign("test.example.com", "00:AA", allow_alt_names=True) self.mocked_master_host.run_sync.assert_has_calls([ mock.call( "puppet ca --disable_warnings deprecations --render-as json " r'list --all --subject "^test\.example\.com$"', is_safe=True, print_output=False, print_progress_bars=False, ), mock.call( "puppet cert --disable_warnings deprecations sign --allow-dns-alt-names test.example.com", print_output=False, print_progress_bars=False, ), mock.call( "puppet ca --disable_warnings deprecations --render-as json " r'list --all --subject "^test\.example\.com$"', is_safe=True, print_output=False, print_progress_bars=False, ), ])
def test_using_sudo_prepends_when_command_is_string(self, mocked_transport_new): """Test that using sudo prepends when command is string.""" nodes_a = "host1" mock_cumin(self.mocked_transports, 0, retvals=[[(nodes_a, b"")]]) mocked_worker = mock.MagicMock() mocked_worker.execute.return_value = 0 mocked_transport_new.return_value = mocked_worker remote.RemoteHosts(self.config, NodeSet(nodes_a), dry_run=False, use_sudo=True).run_sync("command") assert mocked_worker.commands == ["sudo -i command"]
def test_wait_since_failed_execution(self, mocked_sleep): """It should raise PuppetHostsCheckError if fails to get the successful Puppet run within the timeout.""" self.mocked_remote_hosts.run_sync.side_effect = RemoteExecutionError( 1, "fail") self.mocked_remote_hosts.hosts = NodeSet("test.example.com") with pytest.raises(puppet.PuppetHostsCheckError, match="Unable to find a successful Puppet run"): self.puppet_hosts.wait_since(datetime.utcnow()) assert mocked_sleep.called
def test_get_certificate_metadata_raises(self, json_output, exception_message): """It should raise PuppetMasterError if the Puppet CA returns multiple certificates metadata.""" results = [( NodeSet("puppetmaster.example.com"), MsgTreeElem(json_output, parent=MsgTreeElem()), )] self.mocked_master_host.run_sync.return_value = iter(results) with pytest.raises(puppet.PuppetMasterError, match=exception_message): self.puppet_master.get_certificate_metadata("test.example.com")
def test_get_ca_servers_handles_multiple_results(self): """Test test get ca servers handles multiple results.""" self.mocked_remote_hosts.run_sync.return_value = [ ( NodeSet("test0.example.com"), MsgTreeElem(b"test0.puppetmast.er", parent=MsgTreeElem()), ), ( NodeSet("test1.example.com"), MsgTreeElem(b"test1.puppetmast.er", parent=MsgTreeElem()), ), ] result = self.puppet_hosts.get_ca_servers() self.mocked_remote_hosts.run_sync.assert_called_once() assert "test0.example.com" in result assert result["test0.example.com"] == "test0.puppetmast.er" assert "test1.example.com" in result assert result["test1.example.com"] == "test1.puppetmast.er"
def _get_disabled(self) -> Dict[bool, NodeSet]: """Check if Puppet is disabled on the hosts. Returns: dict: a dict with :py:class:`bool` keys for Puppet disabled or not and hosts :py:class:`ClusterShell.NodeSet.NodeSet` as values. """ results = self._remote_hosts.run_sync( f'source {PUPPET_COMMON_SCRIPT} && test -f "${{PUPPET_DISABLEDLOCK}}" && echo "1" || echo "0"', is_safe=True, print_output=False, print_progress_bars=False, ) disabled = {True: NodeSet(), False: NodeSet()} for nodeset, output in results: result = bool(int(output.message().decode().strip())) disabled[result] |= nodeset return disabled