def main():
"""
Parse argument as command and execute that command with
parameters containing the state of MySQL, ContainerPilot, etc.
Default behavior is to run `pre_start` DB initialization.
"""
if len(sys.argv) == 1:
consul = Consul(envs={'CONSUL': os.environ.get('CONSUL', 'consul')})
cmd = pre_start
else:
consul = Consul()
try:
cmd = globals()[sys.argv[1]]
except KeyError:
log.error('Invalid command: %s', sys.argv[1])
sys.exit(1)
my = MySQL()
manta = Manta()
cp = ContainerPilot()
cp.load()
node = Node(mysql=my, consul=consul, manta=manta, cp=cp)
cmd(node)
def test_failover_locked_another_node_is_primary(self):
"""
Given another node is running a failover, wait for that failover.
Given this this node is not marked primary, the node will not
update its ContainerPilot config.
"""
def query_results(*args, **kwargs):
yield ()
yield () # and after two hits we've set up replication
yield [{
'Master_Server_Id': 'node2',
'Master_Host': '192.168.1.102'
}]
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(side_effect=query_results())
self.node.mysql.failover = mock.MagicMock()
def consul_get_primary_results(*args, **kwargs):
yield UnknownPrimary()
yield UnknownPrimary()
yield ('node2', '192.168.1.102')
def lock_sequence(*args, **kwargs):
yield True
yield False
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.put = mock.MagicMock()
self.node.consul.get_primary = mock.MagicMock(
side_effect=consul_get_primary_results())
self.node.consul.lock_failover = mock.MagicMock(return_value=False)
self.node.consul.unlock_failover = mock.MagicMock()
self.node.consul.is_locked = mock.MagicMock(
side_effect=lock_sequence())
with mock.patch('time.sleep'): # cuts 3 sec from test run
manage.on_change(self.node)
self.assertEqual(self.node.consul.get_primary.call_count, 2)
self.node.consul.lock_failover.assert_called_once()
self.assertFalse(self.node.consul.client.health.service.called)
self.assertFalse(self.node.consul.unlock_failover.called)
self.assertFalse(self.node.consul.put.called)
self.assertFalse(self.node.cp.reload.called)
self.assertEqual(self.node.cp.state, REPLICA)
def test_backup_already_running(self):
""" Don't snapshot if there's already a snapshot running """
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.client.session.create.return_value = 'xyzzy'
with mock.patch('manage.write_snapshot') as ws:
lockfile_name = '/tmp/' + BACKUP_LOCK_KEY
try:
backup_lock = open(lockfile_name, 'w')
fcntl.flock(backup_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
manage.snapshot_task(self.node)
finally:
fcntl.flock(backup_lock, fcntl.LOCK_UN)
backup_lock.close()
self.assertFalse(ws.called)
def test_backup_unlocked(self):
"""
Make sure that if a snapshot has run that we unlock correctly.
"""
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.client.session.create.return_value = 'xyzzy'
with mock.patch('manage.write_snapshot') as ws:
lockfile_name = '/tmp/' + BACKUP_LOCK_KEY
try:
backup_lock = open(lockfile_name, 'w')
fcntl.flock(backup_lock, fcntl.LOCK_EX | fcntl.LOCK_NB)
manage.snapshot_task(self.node)
finally:
fcntl.flock(backup_lock, fcntl.LOCK_UN)
backup_lock.close()
manage.snapshot_task(self.node)
self.assertTrue(ws.called)
def test_pre_start_snapshot_incomplete(self):
"""
Given a snapshot that has been marked successful but not
completed, a new node will wait and not crash.
"""
self.node.consul = Consul(get_environ())
self.node.consul.client = mock.MagicMock()
def kv_gets(*args, **kwargs):
yield pyconsul.ConsulException()
yield [0, {'Value': '{"id": "xxxx", "dt": "yyyyy"}'}]
self.node.consul.client.kv.get.side_effect = kv_gets()
manage.pre_start(self.node)
self.node.manta.get_backup.assert_called_once()
self.assertEqual(self.node.consul.client.kv.get.call_count, 2)
self.node.mysql.restore_from_snapshot.assert_called_once()
self.assertFalse(self.node.mysql.initialize_db.called)
def test_replica_first_pass_primary_lockout(self):
"""
Given uninitialized node w/ no primary, then a health primary
retry setting up as a replica
"""
self.node.mysql.wait_for_connection.return_value = True
self.node.mysql.get_primary.side_effect = UnknownPrimary()
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.mark_as_primary = mock.MagicMock(return_value=False)
self.node.consul.client.health.service.return_value = ()
try:
logging.getLogger().setLevel(logging.CRITICAL) # noisy
manage.health(self.node)
self.fail('Should have exited but did not.')
except SystemExit:
pass
self.assertEqual(self.node.cp.state, UNASSIGNED)
def test_replica_first_pass(self):
"""
Given uninitialized node w/ a health primary, set up replication.
"""
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock()
def query_results(*args, **kwargs):
yield ()
yield () # and after two hits we've set up replication
yield [{
'Master_Server_Id': 'node2',
'Master_Host': '192.168.1.102'
}]
self.node.mysql.query.side_effect = query_results()
self.node.mysql.wait_for_connection = mock.MagicMock(return_value=True)
self.node.mysql.setup_replication = mock.MagicMock(return_value=True)
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.client.health.service.return_value = [
0, [{
'Service': {
'ID': 'node2',
'Address': '192.168.1.102'
},
}]
]
manage.health(self.node)
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('show slave status')
]
self.node.mysql.query.assert_has_calls(calls)
self.assertEqual(self.node.consul.client.health.service.call_count, 2)
manage.write_snapshot.assert_called_once()
self.assertEqual(self.node.cp.state, REPLICA)
def test_primary_no_replicas(self):
""" Health check if previously initialized but with no replicas """
os.mkdir(self.LOCK_PATH, 0700)
self.node.mysql = MySQL(envs=get_environ())
self.node.mysql._conn = mock.MagicMock()
self.node.mysql.query = mock.MagicMock(return_value=())
self.node.consul = Consul(envs=get_environ())
self.node.consul.client = mock.MagicMock()
self.node.consul.renew_session = mock.MagicMock()
self.node.consul.client.health.service.return_value = [0, [{
'Service' : {'ID': 'node1', 'Address': '192.168.1.101'},
}]]
manage.health(self.node)
calls = [
mock.call.query('show slave status'),
mock.call.query('show slave hosts'),
mock.call.query('select 1')
]
self.node.mysql.query.assert_has_calls(calls)
self.node.consul.client.health.service.assert_called_once()
self.node.consul.renew_session.assert_called_once()
self.assertEqual(self.node.cp.state, PRIMARY)
def test_parse_with_consul_agent(self):
self.environ['CONSUL_AGENT'] = '1'
consul = Consul(self.environ)
self.assertEqual(consul.host, 'localhost')
