def test_parses_result_with_one_member(self): """Test that parses result with one member.""" expected_node = { "member_id": "415090d15def9053", "name": "toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud", "status": "up", "isLeader": True, "peerURLs": "https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380", "clientURLs": "https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379", } mock_run_sync = _get_mock_run_sync( return_value=b""" 415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true """, # noqa: E501 ) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): gotten_result = controller.get_cluster_info() mock_run_sync.assert_called_once() assert len(gotten_result) == 1 assert expected_node["member_id"] in gotten_result assert gotten_result[expected_node["member_id"]] == expected_node
def test_check_core_masters_in_sync_fail_heartbeat(self, mocked_sleep): """Should raise MysqlLegacyError if unable to get the heartbeat from the current master.""" self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001")) mock_cumin(self.mocked_transports, 0, retvals=[]) with pytest.raises(mysql_legacy.MysqlLegacyError, match="Unable to get heartbeat from master"): self.mysql.check_core_masters_in_sync("eqiad", "codfw") assert not mocked_sleep.called
def test_get_core_dbs_fail_sanity_check(self): """It should raise MysqlLegacyError if matching an invalid number of hosts when looking for masters.""" self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001")) with pytest.raises(mysql_legacy.MysqlLegacyError, match="Matched 1 masters, expected 12"): self.mysql.get_core_dbs(datacenter="eqiad", replication_role="master") assert self.mocked_remote.query.called
def test_check_core_masters_in_sync_not_in_sync(self, mocked_sleep): """Should raise MysqlLegacyError if a master is not in sync with the one in the other DC.""" hosts = NodeSet(EQIAD_CORE_MASTERS_QUERY) self.mocked_remote.query.side_effect = [RemoteHosts(self.config, NodeSet(host)) for host in hosts] + [ RemoteHosts(self.config, NodeSet("db1001")) ] * 3 retvals = [[(host, b"2018-09-06T10:00:00.000000")] for host in hosts] # first heartbeat retvals += [[("db1001", b"2018-09-06T10:00:00.000000")]] * 3 # 3 failed retries of second heartbeat mock_cumin(self.mocked_transports, 0, retvals=retvals) with pytest.raises( mysql_legacy.MysqlLegacyError, match=r"Heartbeat from master db1001 for section .* not yet in sync", ): self.mysql.check_core_masters_in_sync("eqiad", "codfw") assert mocked_sleep.called
def test_raises_if_more_than_one_node_is_used(self): """Test that raises if more than one node is used.""" nodes = RemoteHosts(config=mock.MagicMock(specset=Config), hosts=NodeSet("test[0,1].local.host")) with self.assertRaises(TooManyHosts): EtcdctlController(remote_host=nodes)
def test_adds_the_member_if_not_there(self): """Test that adds the member if not there.""" new_member_fqdn = "i.already.exist" new_member_peer_url = f"https://{new_member_fqdn}:1234" expected_member_id = "1234556789012345" mock_run_sync = _get_mock_run_sync(side_effect=[ b""" 415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true """, # noqa: E501 b"""Added :)""", f""" 415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=false {expected_member_id}: name={new_member_fqdn} peerURLs={new_member_peer_url} """.encode(), # noqa: E501 ]) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): gotten_member_id = controller.ensure_node_exists( new_member_fqdn=new_member_fqdn, member_peer_url=new_member_peer_url, ) _assert_called_with_single_param(param="add", mock_obj=mock_run_sync) assert gotten_member_id == expected_member_id
def test_check_core_masters_in_sync_ok(self, mocked_sleep): """Should check that all core masters are in sync with the master in the other DC.""" hosts = NodeSet(EQIAD_CORE_MASTERS_QUERY) self.mocked_remote.query.side_effect = [RemoteHosts(self.config, NodeSet(host)) for host in hosts] * 2 retvals = [[(host, b"2018-09-06T10:00:00.000000")] for host in hosts] # first heartbeat retvals += [[(host, b"2018-09-06T10:00:01.000000")] for host in hosts] # second heartbeat mock_cumin(self.mocked_transports, 0, retvals=retvals) self.mysql.check_core_masters_in_sync("eqiad", "codfw") assert not mocked_sleep.called
def setup_method(self, _, mocked_transports): """Setup the test environment.""" # pylint: disable=attribute-defined-outside-init self.config = Config(get_fixture_path("remote", "config.yaml")) self.mocked_transports = mocked_transports self.mysql_remote_hosts = mysql_legacy.MysqlLegacyRemoteHosts( RemoteHosts(self.config, NodeSet("host[1-9]"), dry_run=False) ) self.expected = [(NodeSet("host1"), "output1")]
def test_get_core_masters_heartbeats_wrong_data(self): """Should raise MysqlLegacyError if unable to convert the heartbeat into a datetime.""" self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001")) mock_cumin( self.mocked_transports, 0, retvals=[[("db1001", b"2018-09-06-10:00:00.000000")]], ) with pytest.raises(mysql_legacy.MysqlLegacyError, match="Unable to convert heartbeat"): self.mysql.get_core_masters_heartbeats("eqiad", "codfw")
def test_verify_core_masters_readonly_fail(self): """Should raise MysqlLegacyError if some masters do not have the intended read-only value.""" self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db10[01-11]")) mock_cumin( self.mocked_transports, 0, retvals=[[("db1001", b"0"), ("db10[02-11]", b"1")]], ) with pytest.raises( mysql_legacy.MysqlLegacyError, match="Verification failed that core DB masters", ): self.mysql.verify_core_masters_readonly("eqiad", True)
def __new__(cls, icinga_host: RemoteHosts, *, config_file: str = "/etc/icinga/icinga.cfg") -> "CommandFile": """Get the Icinga host command file where to write the commands and cache it. Arguments: icinga_host (spicerack.remote.RemoteHosts): the Icinga host instance. config_file (str, optional): the Icinga configuration file to check for the command file directive. Returns: str: the Icinga command file path on the Icinga host. Raises: spicerack.icinga.IcingaError: if unable to get the command file path. """ # Can't use functools cache decorators because NodeSet are not hashable if len(icinga_host) != 1: raise IcingaError( f"Icinga host must match a single host, got: {icinga_host}") identifier = (str(icinga_host), config_file) if identifier in cls._command_files: return cast(CommandFile, cls._command_files[identifier]) try: # Get the command_file value in the Icinga configuration. command = r"grep -P '\s*command_file\s*=.+' " + config_file command_file = "" for _, output in icinga_host.run_sync( command, is_safe=True, print_output=False, print_progress_bars=False): # Read only operation command_file = output.message().decode().split("=", 1)[-1].strip() if not command_file: raise ValueError( f"Empty or no value found for command_file configuration in {config_file}" ) except (SpicerackError, ValueError) as e: raise IcingaError( f"Unable to read command_file configuration in {config_file}" ) from e cls._command_files[identifier] = command_file return cast(CommandFile, command_file)
def test_passes_correct_key_file(self): """Test that passes correct key file by default.""" expected_key_file = "/etc/etcd/ssl/test0.local.host.priv" mock_run_sync = _get_mock_run_sync(return_value=b"") controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): controller.get_cluster_info() _assert_called_with_single_param( param=f"--key-file {expected_key_file}", mock_obj=mock_run_sync, )
def test_passes_correct_endpoints(self): """Test that passes correct endpoints by default.""" expected_endpoints = "https://test0.local.host:2379" mock_run_sync = _get_mock_run_sync(return_value=b"") controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): controller.get_cluster_info() _assert_called_with_single_param( param=f"--endpoints {expected_endpoints}", mock_obj=mock_run_sync, )
def test_raises_when_no_global_cluster_health(self): """Test that parses cluster global status when unhealthy.""" mock_run_sync = _get_mock_run_sync( return_value=b""" member 5208bbf5c00e7cdf is unhealthy: got unhealthy result from https://toolsbeta-test-k8s-etcd-6.toolsbeta.eqiad1.wikimedia.cloud:2379 """, # noqa: E501 ) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): with self.assertRaises(UnableToParseOutput): controller.get_cluster_health() mock_run_sync.assert_called_once()
def test_raises_when_getting_member_without_id(self): """Test that raises when getting member without id.""" mock_run_sync = _get_mock_run_sync(return_value=b""" 415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true peerURLs=https://idontexist.localhost:1234 """ # noqa: E501 ) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with self.assertRaises(UnableToParseOutput): with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): controller.get_cluster_info()
def test_parses_result_with_one_member(self): """Test that parses result with one member.""" expected_members = {"415090d15def9053": HealthStatus.healthy} mock_run_sync = _get_mock_run_sync( return_value=b""" member 415090d15def9053 is healthy: got healthy result from https://toolsbeta-test-k8s-etcd-6.toolsbeta.eqiad1.wikimedia.cloud:2379 cluster is healthy """, # noqa: E501 ) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): gotten_result = controller.get_cluster_health() mock_run_sync.assert_called_once() assert gotten_result.members_status == expected_members
def test_gets_cluster_unhealthy(self): """Test that parses cluster global status when unhealthy.""" expected_global_status = HealthStatus.unhealthy mock_run_sync = _get_mock_run_sync( return_value=b""" member 5208bbf5c00e7cdf is unhealthy: got unhealthy result from https://toolsbeta-test-k8s-etcd-6.toolsbeta.eqiad1.wikimedia.cloud:2379 cluster is unhealthy """, # noqa: E501 ) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): gotten_result = controller.get_cluster_health() mock_run_sync.assert_called_once() assert expected_global_status == gotten_result.global_status
def test_passes_correct_ca_file(self): """Test that passes correct ca file by default.""" expected_ca_file = "/etc/etcd/ssl/ca.pem" mock_run_sync = _get_mock_run_sync( return_value=b""" member 415090d15def9053 is healthy: got healthy result from https://toolsbeta-test-k8s-etcd-6.toolsbeta.eqiad1.wikimedia.cloud:2379 cluster is healthy """, # noqa: E501 ) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): controller.get_cluster_health() _assert_called_with_single_param( param=f"--ca-file {expected_ca_file}", mock_obj=mock_run_sync, )
def test_skips_removal_if_member_does_not_exist(self): """Test that skips removal if member does not exist.""" non_existing_member_fqdn = "i.dont.exist" expected_result = None mock_run_sync = _get_mock_run_sync(return_value=""" 415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true """.encode() # noqa: E501 ) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): gotten_result = controller.ensure_node_does_not_exist( member_fqdn=non_existing_member_fqdn, ) _assert_called_with_single_param(param="list", mock_obj=mock_run_sync) _assert_not_called_with_single_param(param="remove", mock_obj=mock_run_sync) assert gotten_result == expected_result
def test_uses_default_member_url_if_not_passed(self): """Test that uses default member url if not passed.""" new_member_fqdn = "i.already.exist" expected_peer_url = f"https://{new_member_fqdn}:2380" mock_run_sync = _get_mock_run_sync(side_effect=[ b"", b"""Added :)""", f""" 415090d15def9053: name={new_member_fqdn} peerURLs={expected_peer_url} """.encode(), ]) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): controller.ensure_node_exists(new_member_fqdn=new_member_fqdn) _assert_called_with_single_param(param=expected_peer_url, mock_obj=mock_run_sync)
def test_parses_result_with_member_down(self): """Test that parses result with member down.""" expected_result = { "415090d15def9053": { "member_id": "415090d15def9053", "name": "toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud", "status": "up", "isLeader": True, "peerURLs": "https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380", "clientURLs": "https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379", }, "cf612c785df58f6a": { "member_id": "cf612c785df58f6a", "peerURLs": "https://idontexist.localhost:1234", "status": "unstarted", }, } mock_run_sync = _get_mock_run_sync(return_value=b""" 415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true cf612c785df58f6a[unstarted]: peerURLs=https://idontexist.localhost:1234 """ # noqa: E501 ) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): gotten_result = controller.get_cluster_info() mock_run_sync.assert_called_once() assert expected_result == gotten_result
def test_removes_the_member_if_there_already(self): """Test that it removes the member if there already.""" member_fqdn = "i.already.exist" expected_member_id = "1234556789012345" mock_run_sync = _get_mock_run_sync(side_effect=[ f""" 415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=false {expected_member_id}: name={member_fqdn} peerURLs=http://some.url """.encode(), # noqa: E501 "Removed :)", ]) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): gotten_result = controller.ensure_node_does_not_exist( member_fqdn=member_fqdn) _assert_called_with_single_param(param="remove", mock_obj=mock_run_sync) assert gotten_result == expected_member_id
def remove_configuration(self, configuration: DHCPConfiguration, force: bool = False) -> None: """Remove configuration from target DHCP server then call refresh_dhcp. This will fail if contents do not match unless force is True. Arguments: configuration (spicerack.dhcp.DHCPConfiguration): An instance which provides content and filename for a configuration. force (bool, default False): If set to True, will remove filename regardless. """ if not force: confsha256 = sha256(str(configuration).encode()).hexdigest() try: results = self._hosts.run_sync( f"sha256sum {DHCP_TARGET_PATH}/{configuration.filename}", is_safe=True, print_output=False, print_progress_bars=False, ) except RemoteExecutionError as exc: raise DHCPError(f"Can't test {configuration.filename} for removal.") from exc seen_match = False for _, result in RemoteHosts.results_to_list(results): remotesha256 = result.strip().split()[0] if remotesha256 != confsha256 and not self._dry_run: raise DHCPError(f"Remote {configuration.filename} has a mismatched SHA256, refusing to remove.") seen_match = True if not seen_match: raise DHCPError("Did not get any result trying to get SHA256, refusing to attempt to remove.") try: self._hosts.run_sync( f"/bin/rm -v {DHCP_TARGET_PATH}/{configuration.filename}", print_output=False, print_progress_bars=False ) except RemoteExecutionError as exc: raise DHCPError(f"Can't remove {configuration.filename}.") from exc self.refresh_dhcp()
def test_skips_addition_if_member_already_exists(self): """Test that skips addition if member already exists.""" existing_member_fqdn = "i.already.exist" existing_member_peer_url = f"https://{existing_member_fqdn}:1234" expected_member_id = "1234556789012345" mock_run_sync = _get_mock_run_sync(return_value=f""" 415090d15def9053: name=toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud peerURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2380 clientURLs=https://toolsbeta-test-k8s-etcd-9.toolsbeta.eqiad1.wikimedia.cloud:2379 isLeader=true {expected_member_id}: name={existing_member_fqdn} peerURLs={existing_member_peer_url} """.encode() # noqa: E501 ) controller = EtcdctlController(remote_host=RemoteHosts( config=mock.MagicMock(specset=Config), hosts=NodeSet("test0.local.host")), ) with mock.patch.object(RemoteHosts, "run_sync", mock_run_sync): gotten_member_id = controller.ensure_node_exists( new_member_fqdn=existing_member_fqdn, member_peer_url=existing_member_peer_url, ) _assert_called_with_single_param(param="list", mock_obj=mock_run_sync) _assert_not_called_with_single_param(param="add", mock_obj=mock_run_sync) assert gotten_member_id == expected_member_id
def test_get_core_dbs_ok(self, kwargs, query, match): """It should return the right DBs based on the parameters.""" self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet(match)) self.mysql.get_core_dbs(**kwargs) self.mocked_remote.query.assert_called_once_with(query)
def test_set_core_masters_readonly(self, mode, value, caplog): """It should set the masters as read-only/read-write.""" self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db10[01-11]")) mock_cumin(self.mocked_transports, 0, retvals=[[("db10[01-11]", value)]]) getattr(self.mysql, "set_core_masters_" + mode)("eqiad") assert "SET GLOBAL read_only=" + value.decode() in caplog.text
def test_verify_core_masters_readonly_ok(self, readonly, reply, caplog): """Should verify that the masters have the intended read-only value.""" self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db10[01-11]")) mock_cumin(self.mocked_transports, 0, retvals=[[("db10[01-11]", reply)]]) self.mysql.verify_core_masters_readonly("eqiad", readonly) assert "SELECT @@global.read_only" in caplog.text