def main(): """ Parse argument as command and execute that command with parameters containing the state of MySQL, ContainerPilot, etc. Default behavior is to run `pre_start` DB initialization. """ if len(sys.argv) == 1: consul = Consul(envs={'CONSUL': os.environ.get('CONSUL', 'consul')}) cmd = pre_start else: consul = Consul() try: cmd = globals()[sys.argv[1]] except KeyError: log.error('Invalid command: %s', sys.argv[1]) sys.exit(1) my = MySQL() manta = Manta() cp = ContainerPilot() cp.load() node = Node(mysql=my, consul=consul, manta=manta, cp=cp) cmd(node)
def test_failover_locked_another_node_is_primary(self): """ Given another node is running a failover, wait for that failover. Given this this node is not marked primary, the node will not update its ContainerPilot config. """ def query_results(*args, **kwargs): yield () yield () # and after two hits we've set up replication yield [{ 'Master_Server_Id': 'node2', 'Master_Host': '192.168.1.102' }] self.node.mysql = MySQL(envs=get_environ()) self.node.mysql._conn = mock.MagicMock() self.node.mysql.query = mock.MagicMock(side_effect=query_results()) self.node.mysql.failover = mock.MagicMock() def consul_get_primary_results(*args, **kwargs): yield UnknownPrimary() yield UnknownPrimary() yield ('node2', '192.168.1.102') def lock_sequence(*args, **kwargs): yield True yield False self.node.consul = Consul(envs=get_environ()) self.node.consul.client = mock.MagicMock() self.node.consul.put = mock.MagicMock() self.node.consul.get_primary = mock.MagicMock( side_effect=consul_get_primary_results()) self.node.consul.lock_failover = mock.MagicMock(return_value=False) self.node.consul.unlock_failover = mock.MagicMock() self.node.consul.is_locked = mock.MagicMock( side_effect=lock_sequence()) with mock.patch('time.sleep'): # cuts 3 sec from test run manage.on_change(self.node) self.assertEqual(self.node.consul.get_primary.call_count, 2) self.node.consul.lock_failover.assert_called_once() self.assertFalse(self.node.consul.client.health.service.called) self.assertFalse(self.node.consul.unlock_failover.called) self.assertFalse(self.node.consul.put.called) self.assertFalse(self.node.cp.reload.called) self.assertEqual(self.node.cp.state, REPLICA)
def test_backup_already_running(self): """ Don't snapshot if there's already a snapshot running """ self.node.consul = Consul(envs=get_environ()) self.node.consul.client = mock.MagicMock() self.node.consul.client.session.create.return_value = 'xyzzy' with mock.patch('manage.write_snapshot') as ws: lockfile_name = '/tmp/' + BACKUP_LOCK_KEY try: backup_lock = open(lockfile_name, 'w') fcntl.flock(backup_lock, fcntl.LOCK_EX | fcntl.LOCK_NB) manage.snapshot_task(self.node) finally: fcntl.flock(backup_lock, fcntl.LOCK_UN) backup_lock.close() self.assertFalse(ws.called)
def test_backup_unlocked(self): """ Make sure that if a snapshot has run that we unlock correctly. """ self.node.consul = Consul(envs=get_environ()) self.node.consul.client = mock.MagicMock() self.node.consul.client.session.create.return_value = 'xyzzy' with mock.patch('manage.write_snapshot') as ws: lockfile_name = '/tmp/' + BACKUP_LOCK_KEY try: backup_lock = open(lockfile_name, 'w') fcntl.flock(backup_lock, fcntl.LOCK_EX | fcntl.LOCK_NB) manage.snapshot_task(self.node) finally: fcntl.flock(backup_lock, fcntl.LOCK_UN) backup_lock.close() manage.snapshot_task(self.node) self.assertTrue(ws.called)
def test_pre_start_snapshot_incomplete(self): """ Given a snapshot that has been marked successful but not completed, a new node will wait and not crash. """ self.node.consul = Consul(get_environ()) self.node.consul.client = mock.MagicMock() def kv_gets(*args, **kwargs): yield pyconsul.ConsulException() yield [0, {'Value': '{"id": "xxxx", "dt": "yyyyy"}'}] self.node.consul.client.kv.get.side_effect = kv_gets() manage.pre_start(self.node) self.node.manta.get_backup.assert_called_once() self.assertEqual(self.node.consul.client.kv.get.call_count, 2) self.node.mysql.restore_from_snapshot.assert_called_once() self.assertFalse(self.node.mysql.initialize_db.called)
def test_replica_first_pass_primary_lockout(self): """ Given uninitialized node w/ no primary, then a health primary retry setting up as a replica """ self.node.mysql.wait_for_connection.return_value = True self.node.mysql.get_primary.side_effect = UnknownPrimary() self.node.consul = Consul(envs=get_environ()) self.node.consul.client = mock.MagicMock() self.node.consul.mark_as_primary = mock.MagicMock(return_value=False) self.node.consul.client.health.service.return_value = () try: logging.getLogger().setLevel(logging.CRITICAL) # noisy manage.health(self.node) self.fail('Should have exited but did not.') except SystemExit: pass self.assertEqual(self.node.cp.state, UNASSIGNED)
def test_replica_first_pass(self): """ Given uninitialized node w/ a health primary, set up replication. """ self.node.mysql = MySQL(envs=get_environ()) self.node.mysql._conn = mock.MagicMock() self.node.mysql.query = mock.MagicMock() def query_results(*args, **kwargs): yield () yield () # and after two hits we've set up replication yield [{ 'Master_Server_Id': 'node2', 'Master_Host': '192.168.1.102' }] self.node.mysql.query.side_effect = query_results() self.node.mysql.wait_for_connection = mock.MagicMock(return_value=True) self.node.mysql.setup_replication = mock.MagicMock(return_value=True) self.node.consul = Consul(envs=get_environ()) self.node.consul.client = mock.MagicMock() self.node.consul.client.health.service.return_value = [ 0, [{ 'Service': { 'ID': 'node2', 'Address': '192.168.1.102' }, }] ] manage.health(self.node) calls = [ mock.call.query('show slave status'), mock.call.query('show slave hosts'), mock.call.query('show slave status') ] self.node.mysql.query.assert_has_calls(calls) self.assertEqual(self.node.consul.client.health.service.call_count, 2) manage.write_snapshot.assert_called_once() self.assertEqual(self.node.cp.state, REPLICA)
def test_primary_no_replicas(self): """ Health check if previously initialized but with no replicas """ os.mkdir(self.LOCK_PATH, 0700) self.node.mysql = MySQL(envs=get_environ()) self.node.mysql._conn = mock.MagicMock() self.node.mysql.query = mock.MagicMock(return_value=()) self.node.consul = Consul(envs=get_environ()) self.node.consul.client = mock.MagicMock() self.node.consul.renew_session = mock.MagicMock() self.node.consul.client.health.service.return_value = [0, [{ 'Service' : {'ID': 'node1', 'Address': '192.168.1.101'}, }]] manage.health(self.node) calls = [ mock.call.query('show slave status'), mock.call.query('show slave hosts'), mock.call.query('select 1') ] self.node.mysql.query.assert_has_calls(calls) self.node.consul.client.health.service.assert_called_once() self.node.consul.renew_session.assert_called_once() self.assertEqual(self.node.cp.state, PRIMARY)
def test_parse_with_consul_agent(self): self.environ['CONSUL_AGENT'] = '1' consul = Consul(self.environ) self.assertEqual(consul.host, 'localhost')