async def test_master_lock_changes_owner(app):
# if the lock changes owner to someone else, shutdown postgresql and exist
plugins = setup_plugins(app, pg_replication_role='master')
assert app.initialize() == None
plugins.reset_mock()
with patch('time.sleep') as sleep:
with patch('sys.exit') as exit:
app.master_lock_changed('someone else')
assert exit.called_once_with(0)
assert sleep.called_once_with(10)
assert plugins.mock_calls == [
call.pg_replication_role(),
call.pg_replication_role(),
call.pg_stop(),
call.dcs_disconnect(),
call.master_lock_changed('someone else')
]
assert app._master_lock_owner == 'someone else'
# if the lock is owned by us, carry on trucking
plugins.reset_mock()
with patch('time.sleep') as sleep:
with patch('sys.exit') as exit:
app.master_lock_changed(app.my_id)
assert exit.called_once_with(0)
assert sleep.called_once_with(10)
assert plugins.mock_calls == [
call.pg_replication_role(),
call.master_lock_changed('42')
]
assert app._master_lock_owner == app.my_id
async def test_master_lock_changes_owner(app):
# if the lock changes owner to someone else, shutdown postgresql and exist
plugins = setup_plugins(app,
pg_replication_role='master')
assert app.initialize() == None
plugins.reset_mock()
with patch('time.sleep') as sleep:
with patch('sys.exit') as exit:
app.master_lock_changed('someone else')
assert exit.called_once_with(0)
assert sleep.called_once_with(10)
assert plugins.mock_calls == [
call.pg_replication_role(),
call.pg_replication_role(),
call.pg_stop(),
call.dcs_disconnect(),
call.master_lock_changed('someone else')
]
assert app._master_lock_owner == 'someone else'
# if the lock is owned by us, carry on trucking
plugins.reset_mock()
with patch('time.sleep') as sleep:
with patch('sys.exit') as exit:
app.master_lock_changed(app.my_id)
assert exit.called_once_with(0)
assert sleep.called_once_with(10)
assert plugins.mock_calls == [
call.pg_replication_role(),
call.master_lock_changed('42')
]
assert app._master_lock_owner == app.my_id
def test_restart_replica(app, event_loop):
plugins = setup_plugins(app, pg_replication_role='replica')
app.initialize()
plugins.reset_mock()
with patch('time.sleep') as sleep:
app.restart(10)
assert sleep.called_once_with(10)
event_loop.run_forever() # must be stopped by restart()
assert plugins.mock_calls == [
call.pg_replication_role(),
call.dcs_disconnect()
]
def test_restart_replica(app, event_loop):
plugins = setup_plugins(app,
pg_replication_role='replica')
app.initialize()
plugins.reset_mock()
with patch('time.sleep') as sleep:
app.restart(10)
assert sleep.called_once_with(10)
event_loop.run_forever() # must be stopped by restart()
assert plugins.mock_calls == [
call.pg_replication_role(),
call.dcs_disconnect()
]
def test_restart_replica(app):
plugins = setup_plugins(app,
pg_replication_role='replica')
app.initialize()
plugins.reset_mock()
with patch('time.sleep') as sleep:
with patch('sys.exit') as exit:
app.restart(10)
assert exit.called_once_with(0)
assert sleep.called_once_with(10)
assert app._plugins.mock_calls == [
call.pg_replication_role(),
call.dcs_disconnect()
]
async def test_master_lock_broken(app):
plugins = setup_plugins(app, pg_replication_role='master')
assert app.initialize() == None
plugins.reset_mock()
# if the lock is broken, shutdown postgresql and exist
with patch('time.sleep') as sleep:
with patch('sys.exit') as exit:
app.master_lock_changed(None)
assert exit.called_once_with(0)
assert sleep.called_once_with(10)
assert plugins.mock_calls == [
call.pg_replication_role(),
call.pg_replication_role(),
call.pg_stop(),
call.dcs_disconnect(),
call.master_lock_changed(None)
]
assert app._master_lock_owner == None
async def test_master_unhealthy(app):
plugins = setup_plugins(app, pg_replication_role='master')
app.initialize()
plugins.reset_mock()
app.unhealthy('boom', 'It went Boom', can_be_replica=True)
assert plugins.mock_calls == [
call.dcs_set_state({
'host': '127.0.0.1',
'replication_role': 'master',
'health_problems': {
'boom': {
'reason': 'It went Boom',
'can_be_replica': True
}
}
}),
call.pg_replication_role(),
call.dcs_delete_conn_info(),
]
plugins.reset_mock()
# now we should have _handle_unhealthy_master running
with patch('asyncio.sleep') as sleep, patch(
'zgres.deadman.App._stop') as exit, patch(
'time.sleep') as blocking_sleep:
sleeper = FakeSleeper()
sleep.side_effect = sleeper
exit.side_effect = lambda: sleeper.finish()
# there is no replica, so we just sleep and ping the
# DCS to find a willing replica
states = [iter([])]
plugins.dcs_list_state.side_effect = states
await sleeper.next()
assert plugins.mock_calls == [call.dcs_list_state()]
# we add a willing replica
states = [iter([('other', {'willing': 1})])]
plugins.dcs_list_state.side_effect = states
plugins.reset_mock()
await sleeper.next()
assert plugins.mock_calls == [
call.dcs_list_state(),
call.pg_replication_role(),
call.pg_stop(),
call.dcs_disconnect()
]
async def test_master_lock_broken(app):
plugins = setup_plugins(app,
pg_replication_role='master')
assert app.initialize() == None
plugins.reset_mock()
# if the lock is broken, shutdown postgresql and exist
with patch('time.sleep') as sleep:
with patch('sys.exit') as exit:
app.master_lock_changed(None)
assert exit.called_once_with(0)
assert sleep.called_once_with(10)
assert plugins.mock_calls == [
call.pg_replication_role(),
call.pg_replication_role(),
call.pg_stop(),
call.dcs_disconnect(),
call.master_lock_changed(None)
]
assert app._master_lock_owner == None
async def test_master_unhealthy(app):
plugins = setup_plugins(app,
pg_replication_role='master')
app.initialize()
plugins.reset_mock()
app.unhealthy('boom', 'It went Boom', can_be_replica=True)
assert plugins.mock_calls == [
call.dcs_set_state({
'host': '127.0.0.1',
'replication_role': 'master',
'health_problems': {'boom': {'reason': 'It went Boom', 'can_be_replica': True}}}),
call.pg_replication_role(),
call.dcs_delete_conn_info(),
]
plugins.reset_mock()
# now we should have _handle_unhealthy_master running
with patch('asyncio.sleep') as sleep, patch('sys.exit') as exit, patch('time.sleep') as blocking_sleep:
sleeper = FakeSleeper()
sleep.side_effect = sleeper
exit.side_effect = lambda x: sleeper.finish()
# there is no replica, so we just sleep and ping the
# DCS to find a willing replica
states = [iter([])]
plugins.dcs_get_all_state.side_effect = states
plugins.willing_replicas.side_effect = [iter([])]
await sleeper.next()
assert plugins.mock_calls == [
call.dcs_get_all_state(),
call.willing_replicas(states[0])]
# we add a willing replica
states = [iter([])]
plugins.dcs_get_all_state.side_effect = states
plugins.willing_replicas.side_effect = [iter([('other', {})])]
plugins.reset_mock()
await sleeper.next()
assert plugins.mock_calls == [
call.dcs_get_all_state(),
call.willing_replicas(states[0]),
call.pg_replication_role(),
call.pg_stop(),
call.dcs_disconnect()
]
