def test_check_core_masters_in_sync_fail_heartbeat(self, mocked_sleep):
"""Should raise MysqlLegacyError if unable to get the heartbeat from the current master."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001"))
mock_cumin(self.mocked_transports, 0, retvals=[])
with pytest.raises(mysql_legacy.MysqlLegacyError, match="Unable to get heartbeat from master"):
self.mysql.check_core_masters_in_sync("eqiad", "codfw")
assert not mocked_sleep.called
def test_check_core_masters_in_sync_ok(self, mocked_sleep):
"""Should check that all core masters are in sync with the master in the other DC."""
hosts = NodeSet(EQIAD_CORE_MASTERS_QUERY)
self.mocked_remote.query.side_effect = [RemoteHosts(self.config, NodeSet(host)) for host in hosts] * 2
retvals = [[(host, b"2018-09-06T10:00:00.000000")] for host in hosts] # first heartbeat
retvals += [[(host, b"2018-09-06T10:00:01.000000")] for host in hosts] # second heartbeat
mock_cumin(self.mocked_transports, 0, retvals=retvals)
self.mysql.check_core_masters_in_sync("eqiad", "codfw")
assert not mocked_sleep.called
def test_get_core_masters_heartbeats_wrong_data(self):
"""Should raise MysqlLegacyError if unable to convert the heartbeat into a datetime."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db1001"))
mock_cumin(
self.mocked_transports,
0,
retvals=[[("db1001", b"2018-09-06-10:00:00.000000")]],
)
with pytest.raises(mysql_legacy.MysqlLegacyError, match="Unable to convert heartbeat"):
self.mysql.get_core_masters_heartbeats("eqiad", "codfw")
def test_verify_core_masters_readonly_fail(self):
"""Should raise MysqlLegacyError if some masters do not have the intended read-only value."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db10[01-11]"))
mock_cumin(
self.mocked_transports,
0,
retvals=[[("db1001", b"0"), ("db10[02-11]", b"1")]],
)
with pytest.raises(
mysql_legacy.MysqlLegacyError,
match="Verification failed that core DB masters",
):
self.mysql.verify_core_masters_readonly("eqiad", True)
def test_check_core_masters_in_sync_not_in_sync(self, mocked_sleep):
"""Should raise MysqlLegacyError if a master is not in sync with the one in the other DC."""
hosts = NodeSet(EQIAD_CORE_MASTERS_QUERY)
self.mocked_remote.query.side_effect = [RemoteHosts(self.config, NodeSet(host)) for host in hosts] + [
RemoteHosts(self.config, NodeSet("db1001"))
] * 3
retvals = [[(host, b"2018-09-06T10:00:00.000000")] for host in hosts] # first heartbeat
retvals += [[("db1001", b"2018-09-06T10:00:00.000000")]] * 3 # 3 failed retries of second heartbeat
mock_cumin(self.mocked_transports, 0, retvals=retvals)
with pytest.raises(
mysql_legacy.MysqlLegacyError,
match=r"Heartbeat from master db1001 for section .* not yet in sync",
):
self.mysql.check_core_masters_in_sync("eqiad", "codfw")
assert mocked_sleep.called
def test_execute(self):
"""Calling run_query() should run the given query in the target hosts."""
mock_cumin(self.mocked_transports, 0)
results = self.mysql_remote_hosts.run_query("query1")
assert [(host, msg.message().decode()) for host, msg in results] == self.expected
def test_verify_core_masters_readonly_ok(self, readonly, reply, caplog):
"""Should verify that the masters have the intended read-only value."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db10[01-11]"))
mock_cumin(self.mocked_transports, 0, retvals=[[("db10[01-11]", reply)]])
self.mysql.verify_core_masters_readonly("eqiad", readonly)
assert "SELECT @@global.read_only" in caplog.text
def test_set_core_masters_readonly(self, mode, value, caplog):
"""It should set the masters as read-only/read-write."""
self.mocked_remote.query.return_value = RemoteHosts(self.config, NodeSet("db10[01-11]"))
mock_cumin(self.mocked_transports, 0, retvals=[[("db10[01-11]", value)]])
getattr(self.mysql, "set_core_masters_" + mode)("eqiad")
assert "SET GLOBAL read_only=" + value.decode() in caplog.text