def test_primary_no_replicas(self):
""" Health check if previously initialized but with no replicas """
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.renew_session = mock.MagicMock()
self.node.consul.client.health.service.return_value = [
0, [{
'Service': {
'ID': 'node1',
'Address': '192.168.1.101'
},
}]
]
manage.health(self.node)
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('select 1')
]
self.node.mysql.query.assert_has_calls(calls)
self.node.consul.client.health.service.assert_called_once()
self.node.consul.renew_session.assert_called_once()
self.assertEqual(self.node.cp.state, PRIMARY)
def test_primary_first_pass(self):
"""
Given uninitialized node w/ no other instances running,
set up for running as the primary.
"""
self.node.mysql.wait_for_connection.return_value = True
self.node.mysql.get_primary.side_effect = UnknownPrimary()
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.mark_as_primary = mock.MagicMock(return_value=True)
self.node.consul.renew_session = mock.MagicMock()
manage.write_snapshot = mock.MagicMock(return_value=True)
self.node.consul.client.health.service.return_value = ()
try:
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
calls = [
mock.call.setup_root_user(True),
mock.call.create_db(True),
mock.call.create_default_user(True),
mock.call.create_repl_user(True),
mock.call.expire_root_password(True)
]
self.node.mysql.assert_has_calls(calls)
manage.write_snapshot.assert_called_once()
self.assertEqual(self.node.cp.state, PRIMARY)
def test_replica_first_pass(self):
"""
Given uninitialized node w/ a health primary, set up replication.
"""
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock()
def query_results(*args, **kwargs):
yield ()
yield () # and after two hits we've set up replication
yield [{'Master_Server_Id': 'node2', 'Master_Host': '192.168.1.102'}]
self.node.mysql.query.side_effect = query_results()
self.node.mysql.wait_for_connection = mock.MagicMock(return_value=True)
self.node.mysql.setup_replication = mock.MagicMock(return_value=True)
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.client.health.service.return_value = [0, [{
'Service' : {'ID': 'node2', 'Address': '192.168.1.102'},
}]]
manage.health(self.node)
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.consul.client.health.service.call_count, 2)
manage.write_snapshot.assert_called_once()
self.assertEqual(self.node.cp.state, REPLICA)
def test_replica_no_replication(self):
"""
Health check for failure mode where initial replication setup
failed but a primary already exists in Consul.
"""
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.renew_session = mock.MagicMock()
self.node.consul.client.health.service.return_value = [
0, [{
'Service': {
'ID': 'node2',
'Address': '192.168.1.102'
},
}]
]
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertFalse(self.node.consul.renew_session.called)
self.assertEqual(self.node.cp.state, REPLICA)
def test_replica_no_replication(self):
"""
Health check for failure mode where initial replication setup
failed but a primary already exists in Consul.
"""
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.renew_session = mock.MagicMock()
self.node.consul.client.health.service.return_value = [0, [{
'Service' : {'ID': 'node2', 'Address': '192.168.1.102'},
}]]
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertFalse(self.node.consul.renew_session.called)
self.assertEqual(self.node.cp.state, REPLICA)
def test_replica_first_pass(self):
"""
Given uninitialized node w/ a health primary, set up replication.
"""
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock()
def query_results(*args, **kwargs):
yield ()
yield () # and after two hits we've set up replication
yield [{'Master_Server_Id': 'node2', 'Master_Host': '192.168.1.102'}]
self.node.mysql.query.side_effect = query_results()
self.node.mysql.wait_for_connection = mock.MagicMock(return_value=True)
self.node.mysql.setup_replication = mock.MagicMock(return_value=True)
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.client.health.service.return_value = [0, [{
'Service' : {'ID': 'node2', 'Address': '192.168.1.102'},
}]]
manage.health(self.node)
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.consul.client.health.service.call_count, 2)
manage.write_snapshot.assert_called_once()
self.assertEqual(self.node.cp.state, REPLICA)
def test_primary_no_replicas_no_consul_state_fails(self):
"""
Health check if previously initialized but with no replicas
and no Consul state so we'll remain marked UNASSIGNED which
needs to be a failing health check.
"""
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.renew_session = mock.MagicMock()
self.node.consul.client.health.service.return_value = []
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.consul.client.health.service.call_count, 2)
self.assertEqual(self.node.cp.state, UNASSIGNED)
def test_replica_first_pass_replication_setup_fails(self):
"""
Given uninitialized node w/ failed replication setup, fail
"""
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.mysql.wait_for_connection = mock.MagicMock(return_value=True)
self.node.mysql.setup_replication = mock.MagicMock(return_value=True)
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.client.health.service.return_value = [
0, [{
'Service': {
'ID': 'node2',
'Address': '192.168.1.102'
},
}]
]
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.consul.client.health.service.call_count, 2)
manage.write_snapshot.assert_called_once()
self.assertEqual(self.node.cp.state, REPLICA)
def test_primary_first_pass(self):
"""
Given uninitialized node w/ no other instances running,
set up for running as the primary.
"""
self.node.mysql.wait_for_connection.return_value = True
self.node.mysql.get_primary.side_effect = UnknownPrimary()
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.mark_as_primary = mock.MagicMock(return_value=True)
self.node.consul.renew_session = mock.MagicMock()
manage.write_snapshot = mock.MagicMock(return_value=True)
self.node.consul.client.health.service.return_value = ()
try:
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
calls = [
mock.call.setup_root_user(True),
mock.call.create_db(True),
mock.call.create_default_user(True),
mock.call.create_repl_user(True),
mock.call.expire_root_password(True)
]
self.node.mysql.assert_has_calls(calls)
manage.write_snapshot.assert_called_once()
self.assertEqual(self.node.cp.state, PRIMARY)
def test_primary_no_replicas_no_consul_state_fails(self):
"""
Health check if previously initialized but with no replicas
and no Consul state so we'll remain marked UNASSIGNED which
needs to be a failing health check.
"""
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.renew_session = mock.MagicMock()
self.node.consul.client.health.service.return_value = []
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.consul.client.health.service.call_count, 2)
self.assertEqual(self.node.cp.state, UNASSIGNED)
def test_replica_first_pass_replication_setup_fails(self):
"""
Given uninitialized node w/ failed replication setup, fail
"""
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.mysql.wait_for_connection = mock.MagicMock(return_value=True)
self.node.mysql.setup_replication = mock.MagicMock(return_value=True)
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.client.health.service.return_value = [0, [{
'Service' : {'ID': 'node2', 'Address': '192.168.1.102'},
}]]
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.consul.client.health.service.call_count, 2)
manage.write_snapshot.assert_called_once()
self.assertEqual(self.node.cp.state, REPLICA)
def test_primary_typical(self):
""" Typical health check for primary with established replication """
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql.get_primary.return_value = ('node1', '192.168.1.101')
manage.health(self.node)
self.node.consul.renew_session.assert_called_once()
self.node.mysql.query.assert_called_once() # just the select 1
self.assertEqual(self.node.cp.state, PRIMARY)
def test_primary_typical(self):
""" Typical health check for primary with established replication """
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql.get_primary.return_value = ('node1', '192.168.1.101')
manage.health(self.node)
self.node.consul.renew_session.assert_called_once()
self.node.mysql.query.assert_called_once() # just the select 1
self.assertEqual(self.node.cp.state, PRIMARY)
def test_replica_typical(self):
"""
Typical health check for replica with established replication
"""
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=[
{'Master_Server_Id': 'node2', 'Master_Host': '192.168.1.102'}])
manage.health(self.node)
self.assertFalse(self.node.consul.renew_session.called)
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.cp.state, REPLICA)
def test_replica_typical(self):
"""
Typical health check for replica with established replication
"""
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=[
{'Master_Server_Id': 'node2', 'Master_Host': '192.168.1.102'}])
manage.health(self.node)
self.assertFalse(self.node.consul.renew_session.called)
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.cp.state, REPLICA)
def test_replica_first_pass_primary_lockout(self):
"""
Given uninitialized node w/ no primary, then a health primary
retry setting up as a replica
"""
self.node.mysql.wait_for_connection.return_value = True
self.node.mysql.get_primary.side_effect = UnknownPrimary()
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.mark_as_primary = mock.MagicMock(return_value=False)
self.node.consul.client.health.service.return_value = ()
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
self.assertEqual(self.node.cp.state, UNASSIGNED)
def test_replica_first_pass_primary_lockout(self):
"""
Given uninitialized node w/ no primary, then a health primary
retry setting up as a replica
"""
self.node.mysql.wait_for_connection.return_value = True
self.node.mysql.get_primary.side_effect = UnknownPrimary()
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.mark_as_primary = mock.MagicMock(return_value=False)
self.node.consul.client.health.service.return_value = ()
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
self.assertEqual(self.node.cp.state, UNASSIGNED)
def test_primary_no_replicas(self):
""" Health check if previously initialized but with no replicas """
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.renew_session = mock.MagicMock()
self.node.consul.client.health.service.return_value = [0, [{
'Service' : {'ID': 'node1', 'Address': '192.168.1.101'},
}]]
manage.health(self.node)
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('select 1')
]
self.node.mysql.query.assert_has_calls(calls)
self.node.consul.client.health.service.assert_called_once()
self.node.consul.renew_session.assert_called_once()
self.assertEqual(self.node.cp.state, PRIMARY)
