def test_master_bootstrap(app):
plugins = setup_plugins(app,
dcs_get_database_identifier=None,
dcs_lock=True,
pg_get_database_identifier='42')
timeout = app.initialize()
assert plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# check if we have a db identifier set
call.dcs_get_database_identifier(),
# no, ok, init our db
call.pg_initdb(),
# make sure it starts
call.pg_start(),
call.pg_get_database_identifier(),
# lock the database identifier so no-one else gets here
call.dcs_lock('database_identifier'),
# while locked make sure there is no id set in the DCS before we got the lock
call.dcs_get_database_identifier(),
# Make the first backup while locked with no DCS
call.pg_backup(),
# set the database identifier AFTER
call.dcs_set_database_identifier('42')
]
# shut down cleanly and immediately
assert timeout == 0
def test_failed_over_master_start(app):
# A master has failed over and restarted, another master has sucessfully advanced
plugins = setup_plugins(app,
dcs_lock=False,
dcs_get_timeline=2,
pg_get_timeline=1,
pg_replication_role='master')
# sync startup
timeout = app.initialize()
assert plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# compare our id with the id in the DCS
call.dcs_get_database_identifier(),
call.pg_get_database_identifier(),
# check if I am a replica
call.pg_replication_role(),
# no, so check if there is a master
call.dcs_lock('master'),
call.dcs_get_lock_owner('master'),
call.pg_stop(),
# compare our timeline to what's in the DCS
call.pg_get_timeline(),
call.dcs_get_timeline(),
# we're on an older timeline, so reset
call.pg_reset(),
]
# Carry on running afterwards
assert timeout == 5
def test_failed_over_master_start(app):
# A master has failed over and restarted, another master has sucessfully advanced
plugins = setup_plugins(app,
dcs_lock=False,
dcs_get_timeline=2,
pg_get_timeline=1,
pg_replication_role='master')
# sync startup
timeout = app.initialize()
assert plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# compare our id with the id in the DCS
call.dcs_get_database_identifier(),
call.pg_get_database_identifier(),
# check if I am a replica
call.pg_replication_role(),
# no, so check if there is a master
call.dcs_lock('master'),
call.dcs_get_lock_owner('master'),
call.pg_stop(),
# compare our timeline to what's in the DCS
call.pg_get_timeline(),
call.dcs_get_timeline(),
# we're on an older timeline, so reset
call.pg_reset(),
]
# Carry on running afterwards
assert timeout == 5
def test_master_bootstrap(app):
plugins = setup_plugins(app,
dcs_get_database_identifier=None,
dcs_lock=True,
pg_get_database_identifier='42')
timeout = app.initialize()
assert plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# check if we have a db identifier set
call.dcs_get_database_identifier(),
# no, ok, init our db
call.pg_initdb(),
# make sure it starts
call.pg_start(),
call.pg_get_database_identifier(),
# lock the database identifier so no-one else gets here
call.dcs_lock('database_identifier'),
# while locked make sure there is no id set in the DCS before we got the lock
call.dcs_get_database_identifier(),
# Make the first backup while locked with no DCS
call.pg_backup(),
# set the database identifier AFTER
call.dcs_set_database_identifier('42')
]
# shut down cleanly and immediately
assert timeout == 0
async def test_master_start(app):
plugins = setup_plugins(app,
dcs_get_database_identifier='1234',
dcs_lock=True,
pg_replication_role='master',
pg_get_database_identifier='1234')
def start_monitoring():
app.unhealthy('test_monitor', 'Waiting for first check')
plugins.start_monitoring.side_effect = start_monitoring
# sync startup
timeout = app.initialize()
assert plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# compare our id with the id in the DCS
call.dcs_get_database_identifier(),
call.pg_get_database_identifier(),
# check if I am a replica
call.pg_replication_role(),
# no, so check if there is a master
call.dcs_lock('master'),
# no master, so sure the DB is running
call.pg_start(),
# start monitoring
call.start_monitoring(),
call.dcs_watch(
app.master_lock_changed,
app._notify_state,
app._notify_conn_info,
),
call.get_conn_info(),
# set our first state
call.dcs_set_state({
'host': '127.0.0.1',
'replication_role': 'master',
'health_problems': {'test_monitor':
{'can_be_replica': False, 'reason': 'Waiting for first check'}}})
]
# Carry on running afterwards
assert timeout == None
assert app.health_problems == {'test_monitor': {'can_be_replica': False, 'reason': 'Waiting for first check'}}
# Our test monitor becomes healthy
plugins.reset_mock()
app.healthy('test_monitor')
assert plugins.mock_calls == [
call.dcs_set_state({
'host': '127.0.0.1',
'replication_role': 'master',
'health_problems': {}}),
call.pg_replication_role(),
call.dcs_lock('master'),
call.dcs_set_conn_info({'host': '127.0.0.1'}),
]
def test_replica_start(app):
plugins = setup_plugins(app,
dcs_get_database_identifier='1234',
dcs_lock=True,
pg_am_i_replica=True,
pg_get_database_identifier='1234')
app._conn_info['a'] = 'b'
def start_monitoring():
app.unhealthy('test_monitor', 'Waiting for first check')
plugins.start_monitoring.side_effect = start_monitoring
# sync startup
timeout = app.initialize()
assert plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# compare our id with the id in the DCS
call.dcs_get_database_identifier(),
call.pg_get_database_identifier(),
# check if I am a replica
call.pg_am_i_replica(),
# not master, so sure the DB is running
call.pg_start(),
# start monitoring
call.start_monitoring(),
call.dcs_watch(conn_info=None, state=None),
# setup our connection info
call.get_conn_info(),
# set our first state
call.dcs_set_state({
'a': 'b',
'host': '127.0.0.1',
'health_problems': {'test_monitor':
{'can_be_replica': False, 'reason': 'Waiting for first check'}},
})
]
# Carry on running afterwards
assert timeout == None
assert app.health_problems == {'test_monitor': {'can_be_replica': False, 'reason': 'Waiting for first check'}}
# Our test monitor becomes healthy
plugins.reset_mock()
app.healthy('test_monitor')
assert plugins.mock_calls == [
call.dcs_set_state({'health_problems': {},
'a': 'b',
'host': '127.0.0.1',
}),
call.pg_am_i_replica(),
call.dcs_set_conn_info({'a': 'b', 'host': '127.0.0.1'}),
]
def test_replica_bootstrap(app):
plugins = setup_plugins(app,
dcs_get_database_identifier='1234',
pg_get_database_identifier='42')
timeout = app.initialize()
assert app._plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# compare our id with the id in the DCS
call.dcs_get_database_identifier(),
call.pg_get_database_identifier(),
# make sure postgresql is stopped
call.pg_stop(),
# postgresql restore
call.pg_restore(),
call.pg_setup_replication(),
call.pg_am_i_replica()
]
# shut down cleanly and immediately
assert timeout == 0
def test_master_boostrap_fails_to_lock_db_id(app):
plugins = setup_plugins(app,
dcs_get_database_identifier=None,
dcs_lock=False,
pg_get_database_identifier='42')
timeout = app.initialize()
assert plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# check if we have a db identifier set
call.dcs_get_database_identifier(),
# no, ok, init our db
call.pg_initdb(),
# make sure it starts
call.pg_start(),
call.pg_get_database_identifier(),
# lock the database identifier so no-one else gets here
call.dcs_lock('database_identifier')
]
# shut down cleanly
assert timeout == 5
def test_replica_bootstrap(app):
plugins = setup_plugins(app, dcs_get_database_identifier='1234')
plugins.pg_get_database_identifier.side_effect = ['42', '1234']
timeout = app.initialize()
assert plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# compare our id with the id in the DCS
call.dcs_get_database_identifier(),
call.pg_get_database_identifier(),
# make sure postgresql is stopped
call.pg_stop(),
# postgresql restore
call.pg_initdb(),
call.pg_restore(),
call.pg_setup_replication(None),
call.pg_get_database_identifier(),
call.pg_replication_role()
]
# shut down cleanly and immediately
assert timeout == 0
def test_master_boostrap_fails_to_lock_db_id(app):
plugins = setup_plugins(app,
dcs_get_database_identifier=None,
dcs_lock=False,
pg_get_database_identifier='42')
timeout = app.initialize()
assert plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# check if we have a db identifier set
call.dcs_get_database_identifier(),
# no, ok, init our db
call.pg_initdb(),
# make sure it starts
call.pg_start(),
call.pg_get_database_identifier(),
# lock the database identifier so no-one else gets here
call.dcs_lock('database_identifier')
]
# shut down cleanly
assert timeout == 5
def test_replica_bootstrap_fails_sanity_test(app):
plugins = setup_plugins(app,
pg_am_i_replica=False,
dcs_get_database_identifier='1234',
pg_get_database_identifier='42')
timeout = app.initialize()
assert app._plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# compare our id with the id in the DCS
call.dcs_get_database_identifier(),
call.pg_get_database_identifier(),
# make sure postgresql is stopped
call.pg_stop(),
# postgresql restore
call.pg_restore(),
call.pg_setup_replication(),
call.pg_am_i_replica(),
call.pg_reset(),
]
# shut down after 5 seconds to try again
assert timeout == 5
def test_replica_bootstrap_fails_sanity_test(app):
plugins = setup_plugins(app,
pg_replication_role='master',
dcs_get_database_identifier='1234',
pg_get_database_identifier='42')
timeout = app.initialize()
assert plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# compare our id with the id in the DCS
call.dcs_get_database_identifier(),
call.pg_get_database_identifier(),
# make sure postgresql is stopped
call.pg_stop(),
# postgresql restore
call.pg_initdb(),
call.pg_restore(),
call.pg_setup_replication(None),
call.pg_get_database_identifier(),
call.pg_replication_role(),
call.pg_reset(),
]
# shut down after 5 seconds to try again
assert timeout == 5
def test_replica_start(app):
plugins = setup_plugins(app,
dcs_get_database_identifier='1234',
dcs_lock=True,
pg_replication_role='replica',
pg_get_database_identifier='1234')
app._conn_info['a'] = 'b'
def start_monitoring():
app.unhealthy('test_monitor', 'Waiting for first check')
plugins.start_monitoring.side_effect = start_monitoring
# sync startup
timeout = app.initialize()
assert plugins.mock_calls == [
call.initialize(),
call.get_my_id(),
# compare our id with the id in the DCS
call.dcs_get_database_identifier(),
call.pg_get_database_identifier(),
# check if I am a replica
call.pg_replication_role(),
# not master, so sure the DB is running
call.pg_start(),
# start monitoring
call.start_monitoring(),
call.dcs_watch(
app.master_lock_changed,
app._notify_state,
app._notify_conn_info,
),
# setup our connection info
call.get_conn_info(),
# set our first state
call.dcs_set_state({
'a': 'b',
'host': '127.0.0.1',
'replication_role': 'replica',
'health_problems': {
'test_monitor': {
'can_be_replica': False,
'reason': 'Waiting for first check'
}
},
})
]
# Carry on running afterwards
assert timeout == None
assert app.health_problems == {
'test_monitor': {
'can_be_replica': False,
'reason': 'Waiting for first check'
}
}
# Our test monitor becomes healthy
plugins.reset_mock()
with patch('time.time') as mock_time:
app.healthy('test_monitor')
assert plugins.mock_calls == [
call.veto_takeover({
'health_problems': {},
'a': 'b',
'replication_role': 'replica',
'host': '127.0.0.1'
}),
call.dcs_set_state({
'health_problems': {},
'a': 'b',
'replication_role': 'replica',
'host': '127.0.0.1',
'willing': mock_time(),
}),
call.pg_replication_role(),
call.dcs_set_conn_info({
'a': 'b',
'host': '127.0.0.1'
}),
]