async def test_master_lock_changes_owner(app): # if the lock changes owner to someone else, shutdown postgresql and exist plugins = setup_plugins(app, pg_replication_role='master') assert app.initialize() == None plugins.reset_mock() with patch('time.sleep') as sleep: with patch('sys.exit') as exit: app.master_lock_changed('someone else') assert exit.called_once_with(0) assert sleep.called_once_with(10) assert plugins.mock_calls == [ call.pg_replication_role(), call.pg_replication_role(), call.pg_stop(), call.dcs_disconnect(), call.master_lock_changed('someone else') ] assert app._master_lock_owner == 'someone else' # if the lock is owned by us, carry on trucking plugins.reset_mock() with patch('time.sleep') as sleep: with patch('sys.exit') as exit: app.master_lock_changed(app.my_id) assert exit.called_once_with(0) assert sleep.called_once_with(10) assert plugins.mock_calls == [ call.pg_replication_role(), call.master_lock_changed('42') ] assert app._master_lock_owner == app.my_id
def test_restart_replica(app, event_loop): plugins = setup_plugins(app, pg_replication_role='replica') app.initialize() plugins.reset_mock() with patch('time.sleep') as sleep: app.restart(10) assert sleep.called_once_with(10) event_loop.run_forever() # must be stopped by restart() assert plugins.mock_calls == [ call.pg_replication_role(), call.dcs_disconnect() ]
def test_restart_replica(app): plugins = setup_plugins(app, pg_replication_role='replica') app.initialize() plugins.reset_mock() with patch('time.sleep') as sleep: with patch('sys.exit') as exit: app.restart(10) assert exit.called_once_with(0) assert sleep.called_once_with(10) assert app._plugins.mock_calls == [ call.pg_replication_role(), call.dcs_disconnect() ]
async def test_master_lock_broken(app): plugins = setup_plugins(app, pg_replication_role='master') assert app.initialize() == None plugins.reset_mock() # if the lock is broken, shutdown postgresql and exist with patch('time.sleep') as sleep: with patch('sys.exit') as exit: app.master_lock_changed(None) assert exit.called_once_with(0) assert sleep.called_once_with(10) assert plugins.mock_calls == [ call.pg_replication_role(), call.pg_replication_role(), call.pg_stop(), call.dcs_disconnect(), call.master_lock_changed(None) ] assert app._master_lock_owner == None
async def test_master_unhealthy(app): plugins = setup_plugins(app, pg_replication_role='master') app.initialize() plugins.reset_mock() app.unhealthy('boom', 'It went Boom', can_be_replica=True) assert plugins.mock_calls == [ call.dcs_set_state({ 'host': '127.0.0.1', 'replication_role': 'master', 'health_problems': { 'boom': { 'reason': 'It went Boom', 'can_be_replica': True } } }), call.pg_replication_role(), call.dcs_delete_conn_info(), ] plugins.reset_mock() # now we should have _handle_unhealthy_master running with patch('asyncio.sleep') as sleep, patch( 'zgres.deadman.App._stop') as exit, patch( 'time.sleep') as blocking_sleep: sleeper = FakeSleeper() sleep.side_effect = sleeper exit.side_effect = lambda: sleeper.finish() # there is no replica, so we just sleep and ping the # DCS to find a willing replica states = [iter([])] plugins.dcs_list_state.side_effect = states await sleeper.next() assert plugins.mock_calls == [call.dcs_list_state()] # we add a willing replica states = [iter([('other', {'willing': 1})])] plugins.dcs_list_state.side_effect = states plugins.reset_mock() await sleeper.next() assert plugins.mock_calls == [ call.dcs_list_state(), call.pg_replication_role(), call.pg_stop(), call.dcs_disconnect() ]
async def test_master_unhealthy(app): plugins = setup_plugins(app, pg_replication_role='master') app.initialize() plugins.reset_mock() app.unhealthy('boom', 'It went Boom', can_be_replica=True) assert plugins.mock_calls == [ call.dcs_set_state({ 'host': '127.0.0.1', 'replication_role': 'master', 'health_problems': {'boom': {'reason': 'It went Boom', 'can_be_replica': True}}}), call.pg_replication_role(), call.dcs_delete_conn_info(), ] plugins.reset_mock() # now we should have _handle_unhealthy_master running with patch('asyncio.sleep') as sleep, patch('sys.exit') as exit, patch('time.sleep') as blocking_sleep: sleeper = FakeSleeper() sleep.side_effect = sleeper exit.side_effect = lambda x: sleeper.finish() # there is no replica, so we just sleep and ping the # DCS to find a willing replica states = [iter([])] plugins.dcs_get_all_state.side_effect = states plugins.willing_replicas.side_effect = [iter([])] await sleeper.next() assert plugins.mock_calls == [ call.dcs_get_all_state(), call.willing_replicas(states[0])] # we add a willing replica states = [iter([])] plugins.dcs_get_all_state.side_effect = states plugins.willing_replicas.side_effect = [iter([('other', {})])] plugins.reset_mock() await sleeper.next() assert plugins.mock_calls == [ call.dcs_get_all_state(), call.willing_replicas(states[0]), call.pg_replication_role(), call.pg_stop(), call.dcs_disconnect() ]