def main():
"""
Parse argument as command and execute that command with
parameters containing the state of MySQL, ContainerPilot, etc.
Default behavior is to run `pre_start` DB initialization.
"""
if len(sys.argv) == 1:
consul = Consul(envs={'CONSUL': os.environ.get('CONSUL', 'consul')})
cmd = pre_start
else:
consul = Consul()
try:
cmd = globals()[sys.argv[1]]
except KeyError:
log.error('Invalid command: %s', sys.argv[1])
sys.exit(1)
my = MySQL()
snapshot_backend = os.environ.get('SNAPSHOT_BACKEND', 'manta')
if snapshot_backend == 'local':
snaps = Local()
elif snapshot_backend == 'minio':
snaps = Minio()
else:
snaps = Manta()
cp = ContainerPilot()
cp.load()
node = Node(mysql=my, consul=consul, snaps=snaps, cp=cp)
cmd(node)
my.close()
def test_replica_first_pass_replication_setup_fails(self):
"""
Given uninitialized node w/ failed replication setup, fail
"""
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.mysql.wait_for_connection = mock.MagicMock(return_value=True)
self.node.mysql.setup_replication = mock.MagicMock(return_value=True)
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.client.health.service.return_value = [
0, [{
'Service': {
'ID': 'node2',
'Address': '192.168.1.102'
},
}]
]
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.consul.client.health.service.call_count, 2)
manage.write_snapshot.assert_called_once()
self.assertEqual(self.node.cp.state, REPLICA)
def test_replica_no_replication(self):
"""
Health check for failure mode where initial replication setup
failed but a primary already exists in Consul.
"""
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.renew_session = mock.MagicMock()
self.node.consul.client.health.service.return_value = [
0, [{
'Service': {
'ID': 'node2',
'Address': '192.168.1.102'
},
}]
]
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertFalse(self.node.consul.renew_session.called)
self.assertEqual(self.node.cp.state, REPLICA)
def test_primary_no_replicas_no_consul_state_fails(self):
"""
Health check if previously initialized but with no replicas
and no Consul state so we'll remain marked UNASSIGNED which
needs to be a failing health check.
"""
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.renew_session = mock.MagicMock()
self.node.consul.client.health.service.return_value = []
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.consul.client.health.service.call_count, 2)
self.assertEqual(self.node.cp.state, UNASSIGNED)
def test_primary_no_replicas(self):
""" Health check if previously initialized but with no replicas """
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.renew_session = mock.MagicMock()
self.node.consul.client.health.service.return_value = [
0, [{
'Service': {
'ID': 'node1',
'Address': '192.168.1.101'
},
}]
]
manage.health(self.node)
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('select 1')
]
self.node.mysql.query.assert_has_calls(calls)
self.node.consul.client.health.service.assert_called_once()
self.node.consul.renew_session.assert_called_once()
self.assertEqual(self.node.cp.state, PRIMARY)
def test_failover_runs_this_node_is_primary(self):
"""
Given a successful failover where this node is marked primary,
the node will update its ContainerPilot config as required
"""
def query_results(*args, **kwargs):
yield ()
yield () # and after two hits we've set up replication
yield [{
'Master_Server_Id': 'node1',
'Master_Host': '192.168.1.101'
}]
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(side_effect=query_results())
self.node.mysql.failover = mock.MagicMock()
def consul_get_primary_results(*args, **kwargs):
yield UnknownPrimary()
yield UnknownPrimary()
yield ('node1', '192.168.1.101')
self.node.consul.get_primary.side_effect = consul_get_primary_results()
self.node.consul.lock.return_value = True
self.node.consul.read_lock.return_value = None, None
self.node.consul.client.health.service.return_value = [
0,
[{
'Service': {
'ID': 'node1',
'Address': '192.168.1.101'
}
}, {
'Service': {
'ID': 'node3',
'Address': '192.168.1.103'
}
}]
]
manage.on_change(self.node)
self.assertEqual(self.node.consul.get_primary.call_count, 2)
self.node.consul.lock_failover.assert_called_once()
self.node.consul.client.health.service.assert_called_once()
self.assertFalse(self.node.consul.unlock_failover.called)
self.node.consul.put.assert_called_once()
self.node.cp.reload.assert_called_once()
self.assertEqual(self.node.cp.state, PRIMARY)
def test_failover_locked_another_node_is_primary(self):
"""
Given another node is running a failover, wait for that failover.
Given this this node is not marked primary, the node will not
update its ContainerPilot config.
"""
def query_results(*args, **kwargs):
yield ()
yield () # and after two hits we've set up replication
yield [{
'Master_Server_Id': 'node2',
'Master_Host': '192.168.1.102'
}]
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(side_effect=query_results())
self.node.mysql.failover = mock.MagicMock()
def consul_get_primary_results(*args, **kwargs):
yield UnknownPrimary()
yield UnknownPrimary()
yield ('node2', '192.168.1.102')
def lock_sequence(*args, **kwargs):
yield True
yield False
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.put = mock.MagicMock()
self.node.consul.get_primary = mock.MagicMock(
side_effect=consul_get_primary_results())
self.node.consul.lock_failover = mock.MagicMock(return_value=False)
self.node.consul.unlock_failover = mock.MagicMock()
self.node.consul.is_locked = mock.MagicMock(
side_effect=lock_sequence())
with mock.patch('time.sleep'): # cuts 3 sec from test run
manage.on_change(self.node)
self.assertEqual(self.node.consul.get_primary.call_count, 2)
self.node.consul.lock_failover.assert_called_once()
self.assertFalse(self.node.consul.client.health.service.called)
self.assertFalse(self.node.consul.unlock_failover.called)
self.assertFalse(self.node.consul.put.called)
self.assertFalse(self.node.cp.reload.called)
self.assertEqual(self.node.cp.state, REPLICA)
def test_failover_fails(self):
"""
Given a failed failover, ensure we unlock the failover lock
but exit with an unhandled exception without trying to set
status.
"""
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.mysql.failover = mock.MagicMock(
side_effect=Exception('fail'))
self.node.consul.get_primary.side_effect = UnknownPrimary()
self.node.consul.lock_failover.return_value = True
self.node.consul.read_lock.return_value = None, None
self.node.consul.client.health.service.return_value = [
0,
[{
'Service': {
'ID': 'node1',
'Address': '192.168.1.101'
}
}, {
'Service': {
'ID': 'node3',
'Address': '192.168.1.102'
}
}]
]
try:
manage.on_change(self.node)
self.fail('Expected unhandled exception but did not.')
except Exception as ex:
self.assertEqual(ex.message, 'fail')
self.assertEqual(self.node.consul.get_primary.call_count, 2)
self.node.consul.lock_failover.assert_called_once()
self.node.consul.client.health.service.assert_called_once()
self.node.consul.unlock_failover.assert_called_once()
self.assertFalse(self.node.cp.reload.called)
self.assertEqual(self.node.cp.state, UNASSIGNED)
def test_replica_first_pass(self):
"""
Given uninitialized node w/ a health primary, set up replication.
"""
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock()
def query_results(*args, **kwargs):
yield ()
yield () # and after two hits we've set up replication
yield [{
'Master_Server_Id': 'node2',
'Master_Host': '192.168.1.102'
}]
self.node.mysql.query.side_effect = query_results()
self.node.mysql.wait_for_connection = mock.MagicMock(return_value=True)
self.node.mysql.setup_replication = mock.MagicMock(return_value=True)
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.client.health.service.return_value = [
0, [{
'Service': {
'ID': 'node2',
'Address': '192.168.1.102'
},
}]
]
manage.health(self.node)
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.consul.client.health.service.call_count, 2)
manage.write_snapshot.assert_called_once()
self.assertEqual(self.node.cp.state, REPLICA)
def test_replica_typical(self):
"""
Typical health check for replica with established replication
"""
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(
return_value=[{
'Master_Server_Id': 'node2',
'Master_Host': '192.168.1.102'
}])
manage.health(self.node)
self.assertFalse(self.node.consul.renew_session.called)
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.cp.state, REPLICA)
def setUp(self):
logging.getLogger().setLevel(logging.WARN)
self.environ = get_environ()
self.my = MySQL(self.environ)
self.my._conn = mock.MagicMock()
class TestMySQL(unittest.TestCase):
def setUp(self):
logging.getLogger().setLevel(logging.WARN)
self.environ = get_environ()
self.my = MySQL(self.environ)
self.my._conn = mock.MagicMock()
def tearDown(self):
logging.getLogger().setLevel(logging.DEBUG)
def test_parse(self):
self.assertEqual(self.my.mysql_db, 'test_mydb')
self.assertEqual(self.my.mysql_user, 'test_me')
self.assertEqual(self.my.mysql_password, 'test_pass')
self.assertEqual(self.my.mysql_root_password, 'test_root_pass')
self.assertEqual(self.my.mysql_random_root_password, True)
self.assertEqual(self.my.mysql_onetime_password, True)
self.assertEqual(self.my.repl_user, 'test_repl_user')
self.assertEqual(self.my.repl_password, 'test_repl_pass')
self.assertEqual(self.my.datadir, '/var/lib/mysql')
self.assertEqual(self.my.pool_size, 100)
self.assertIsNotNone(self.my.ip)
def test_query_buffer_execute_should_flush(self):
self.my.add('query 1', ())
self.assertEqual(len(self.my._query_buffer.items()), 1)
self.assertEqual(len(self.my._conn.mock_calls), 0)
self.my.execute('query 2', ())
self.assertEqual(len(self.my._query_buffer.items()), 0)
exec_calls = [
mock.call.cursor().execute('query 1', params=()),
mock.call.commit(),
mock.call.cursor().fetchall(),
mock.call.cursor().execute('query 2', params=()),
mock.call.commit(),
mock.call.cursor().fetchall(),
mock.call.cursor().close()
]
self.assertEqual(self.my._conn.mock_calls[2:], exec_calls)
def test_query_buffer_execute_many_should_flush(self):
self.my.add('query 3', ())
self.my.add('query 4', ())
self.my.add('query 5', ())
self.my.execute_many()
self.assertEqual(len(self.my._query_buffer.items()), 0)
exec_many_calls = [
mock.call.cursor().execute('query 3', params=()),
mock.call.commit(),
mock.call.cursor().fetchall(),
mock.call.cursor().execute('query 4', params=()),
mock.call.commit(),
mock.call.cursor().fetchall(),
mock.call.cursor().execute('query 5', params=()),
mock.call.commit(),
mock.call.cursor().fetchall(),
mock.call.cursor().close()
]
self.assertEqual(self.my._conn.mock_calls[2:], exec_many_calls)
def test_query_buffer_query_should_flush(self):
self.my.query('query 6', ())
self.assertEqual(len(self.my._query_buffer.items()), 0)
query_calls = [
mock.call.cursor().execute('query 6', params=()),
mock.call.cursor().fetchall(),
mock.call.cursor().close()
]
self.assertEqual(self.my._conn.mock_calls[2:], query_calls)
def test_expected_setup_statements(self):
conn = mock.MagicMock()
self.my.setup_root_user(conn)
self.my.create_db(conn)
self.my.create_default_user(conn)
self.my.create_repl_user(conn)
self.my.expire_root_password(conn)
self.assertEqual(len(self.my._conn.mock_calls),
0) # use param, not attr
statements = [
args[0] for (name, args, _) in conn.mock_calls
if name == 'cursor().execute'
]
expected = [
'SET @@SESSION.SQL_LOG_BIN=0;',
"DELETE FROM `mysql`.`user` where user != 'mysql.sys';",
'CREATE USER `root`@`%` IDENTIFIED BY %s ;',
'GRANT ALL ON *.* TO `root`@`%` WITH GRANT OPTION ;',
'DROP DATABASE IF EXISTS test ;', 'FLUSH PRIVILEGES ;',
'CREATE DATABASE IF NOT EXISTS `test_mydb`;',
'CREATE USER `test_me`@`%` IDENTIFIED BY %s;',
'GRANT ALL ON `test_mydb`.* TO `test_me`@`%`;',
'FLUSH PRIVILEGES;',
'CREATE USER `test_repl_user`@`%` IDENTIFIED BY %s; ',
('GRANT SUPER, SELECT, INSERT, REPLICATION SLAVE, RELOAD,'
' LOCK TABLES, GRANT OPTION, REPLICATION CLIENT, RELOAD,'
' DROP, CREATE ON *.* TO `test_repl_user`@`%`; '),
'FLUSH PRIVILEGES;', 'ALTER USER `root`@`%` PASSWORD EXPIRE'
]
self.assertEqual(statements, expected)