Пример #1
0
def test_master_bootstrap(app):
    plugins = setup_plugins(app,
            dcs_get_database_identifier=None,
            dcs_lock=True,
            pg_get_database_identifier='42')
    timeout = app.initialize()
    assert plugins.mock_calls ==  [
            call.initialize(),
            call.get_my_id(),
            # check if we have a db identifier set
            call.dcs_get_database_identifier(),
            # no, ok, init our db
            call.pg_initdb(),
            # make sure it starts
            call.pg_start(),
            call.pg_get_database_identifier(),
            # lock the database identifier so no-one else gets here
            call.dcs_lock('database_identifier'),
            # while locked make sure there is no id set in the DCS before we got the lock
            call.dcs_get_database_identifier(),
            # Make the first backup while locked with no DCS
            call.pg_backup(),
            # set the database identifier AFTER
            call.dcs_set_database_identifier('42')
            ]
    # shut down cleanly and immediately
    assert timeout == 0
Пример #2
0
def test_failed_over_master_start(app):
    # A master has failed over and restarted, another master has sucessfully advanced
    plugins = setup_plugins(app,
                            dcs_lock=False,
                            dcs_get_timeline=2,
                            pg_get_timeline=1,
                            pg_replication_role='master')
    # sync startup
    timeout = app.initialize()
    assert plugins.mock_calls == [
        call.initialize(),
        call.get_my_id(),
        # compare our id with the id in the DCS
        call.dcs_get_database_identifier(),
        call.pg_get_database_identifier(),
        # check if I am a replica
        call.pg_replication_role(),
        # no, so check if there is a master
        call.dcs_lock('master'),
        call.dcs_get_lock_owner('master'),
        call.pg_stop(),
        # compare our timeline to what's in the DCS
        call.pg_get_timeline(),
        call.dcs_get_timeline(),
        # we're on an older timeline, so reset
        call.pg_reset(),
    ]
    # Carry on running afterwards
    assert timeout == 5
Пример #3
0
def test_failed_over_master_start(app):
    # A master has failed over and restarted, another master has sucessfully advanced
    plugins = setup_plugins(app,
            dcs_lock=False,
            dcs_get_timeline=2,
            pg_get_timeline=1,
            pg_replication_role='master')
    # sync startup
    timeout = app.initialize()
    assert plugins.mock_calls ==  [
            call.initialize(),
            call.get_my_id(),
            # compare our id with the id in the DCS
            call.dcs_get_database_identifier(),
            call.pg_get_database_identifier(),
            # check if I am a replica
            call.pg_replication_role(),
            # no, so check if there is a master
            call.dcs_lock('master'),
            call.dcs_get_lock_owner('master'),
            call.pg_stop(),
            # compare our timeline to what's in the DCS
            call.pg_get_timeline(),
            call.dcs_get_timeline(),
            # we're on an older timeline, so reset
            call.pg_reset(),
            ]
    # Carry on running afterwards
    assert timeout == 5
Пример #4
0
def test_master_bootstrap(app):
    plugins = setup_plugins(app,
                            dcs_get_database_identifier=None,
                            dcs_lock=True,
                            pg_get_database_identifier='42')
    timeout = app.initialize()
    assert plugins.mock_calls == [
        call.initialize(),
        call.get_my_id(),
        # check if we have a db identifier set
        call.dcs_get_database_identifier(),
        # no, ok, init our db
        call.pg_initdb(),
        # make sure it starts
        call.pg_start(),
        call.pg_get_database_identifier(),
        # lock the database identifier so no-one else gets here
        call.dcs_lock('database_identifier'),
        # while locked make sure there is no id set in the DCS before we got the lock
        call.dcs_get_database_identifier(),
        # Make the first backup while locked with no DCS
        call.pg_backup(),
        # set the database identifier AFTER
        call.dcs_set_database_identifier('42')
    ]
    # shut down cleanly and immediately
    assert timeout == 0
Пример #5
0
async def test_master_start(app):
    plugins = setup_plugins(app,
            dcs_get_database_identifier='1234',
            dcs_lock=True,
            pg_replication_role='master',
            pg_get_database_identifier='1234')
    def start_monitoring():
        app.unhealthy('test_monitor', 'Waiting for first check')
    plugins.start_monitoring.side_effect = start_monitoring
    # sync startup
    timeout = app.initialize()
    assert plugins.mock_calls ==  [
            call.initialize(),
            call.get_my_id(),
            # compare our id with the id in the DCS
            call.dcs_get_database_identifier(),
            call.pg_get_database_identifier(),
            # check if I am a replica
            call.pg_replication_role(),
            # no, so check if there is a master
            call.dcs_lock('master'),
            # no master, so sure the DB is running
            call.pg_start(),
            # start monitoring
            call.start_monitoring(),
            call.dcs_watch(
                app.master_lock_changed,
                app._notify_state,
                app._notify_conn_info,
            ),
            call.get_conn_info(),
            # set our first state
            call.dcs_set_state({
                'host': '127.0.0.1',
                'replication_role': 'master',
                'health_problems': {'test_monitor':
                    {'can_be_replica': False, 'reason': 'Waiting for first check'}}})
            ]
    # Carry on running afterwards
    assert timeout == None
    assert app.health_problems == {'test_monitor': {'can_be_replica': False, 'reason': 'Waiting for first check'}}
    # Our test monitor becomes healthy
    plugins.reset_mock()
    app.healthy('test_monitor')
    assert plugins.mock_calls ==  [
            call.dcs_set_state({
                'host': '127.0.0.1',
                'replication_role': 'master',
                'health_problems': {}}),
            call.pg_replication_role(),
            call.dcs_lock('master'),
            call.dcs_set_conn_info({'host': '127.0.0.1'}),
           ]
Пример #6
0
def test_replica_start(app):
    plugins = setup_plugins(app,
            dcs_get_database_identifier='1234',
            dcs_lock=True,
            pg_am_i_replica=True,
            pg_get_database_identifier='1234')
    app._conn_info['a'] = 'b'
    def start_monitoring():
        app.unhealthy('test_monitor', 'Waiting for first check')
    plugins.start_monitoring.side_effect = start_monitoring
    # sync startup
    timeout = app.initialize()
    assert plugins.mock_calls ==  [
            call.initialize(),
            call.get_my_id(),
            # compare our id with the id in the DCS
            call.dcs_get_database_identifier(),
            call.pg_get_database_identifier(),
            # check if I am a replica
            call.pg_am_i_replica(),
            # not master, so sure the DB is running
            call.pg_start(),
            # start monitoring
            call.start_monitoring(),
            call.dcs_watch(conn_info=None, state=None),
            # setup our connection info
            call.get_conn_info(),
            # set our first state
            call.dcs_set_state({
                'a': 'b',
                'host': '127.0.0.1',
                'health_problems': {'test_monitor':
                    {'can_be_replica': False, 'reason': 'Waiting for first check'}},
                })
            ]
    # Carry on running afterwards
    assert timeout == None
    assert app.health_problems == {'test_monitor': {'can_be_replica': False, 'reason': 'Waiting for first check'}}
    # Our test monitor becomes healthy
    plugins.reset_mock()
    app.healthy('test_monitor')
    assert plugins.mock_calls ==  [
            call.dcs_set_state({'health_problems': {},
                'a': 'b',
                'host': '127.0.0.1',
                }),
            call.pg_am_i_replica(),
            call.dcs_set_conn_info({'a': 'b', 'host': '127.0.0.1'}),
           ]
Пример #7
0
def test_replica_bootstrap(app):
    plugins = setup_plugins(app,
            dcs_get_database_identifier='1234',
            pg_get_database_identifier='42')
    timeout = app.initialize()
    assert app._plugins.mock_calls ==  [
            call.initialize(),
            call.get_my_id(),
            # compare our id with the id in the DCS
            call.dcs_get_database_identifier(),
            call.pg_get_database_identifier(),
            # make sure postgresql is stopped
            call.pg_stop(),
            # postgresql restore
            call.pg_restore(),
            call.pg_setup_replication(),
            call.pg_am_i_replica()
            ]
    # shut down cleanly and immediately
    assert timeout == 0
Пример #8
0
def test_master_boostrap_fails_to_lock_db_id(app):
    plugins = setup_plugins(app,
            dcs_get_database_identifier=None,
            dcs_lock=False,
            pg_get_database_identifier='42')
    timeout = app.initialize()
    assert plugins.mock_calls ==  [
            call.initialize(),
            call.get_my_id(),
            # check if we have a db identifier set
            call.dcs_get_database_identifier(),
            # no, ok, init our db
            call.pg_initdb(),
            # make sure it starts
            call.pg_start(),
            call.pg_get_database_identifier(),
            # lock the database identifier so no-one else gets here
            call.dcs_lock('database_identifier')
            ]
    # shut down cleanly
    assert timeout == 5
Пример #9
0
def test_replica_bootstrap(app):
    plugins = setup_plugins(app, dcs_get_database_identifier='1234')
    plugins.pg_get_database_identifier.side_effect = ['42', '1234']
    timeout = app.initialize()
    assert plugins.mock_calls == [
        call.initialize(),
        call.get_my_id(),
        # compare our id with the id in the DCS
        call.dcs_get_database_identifier(),
        call.pg_get_database_identifier(),
        # make sure postgresql is stopped
        call.pg_stop(),
        # postgresql restore
        call.pg_initdb(),
        call.pg_restore(),
        call.pg_setup_replication(None),
        call.pg_get_database_identifier(),
        call.pg_replication_role()
    ]
    # shut down cleanly and immediately
    assert timeout == 0
Пример #10
0
def test_master_boostrap_fails_to_lock_db_id(app):
    plugins = setup_plugins(app,
                            dcs_get_database_identifier=None,
                            dcs_lock=False,
                            pg_get_database_identifier='42')
    timeout = app.initialize()
    assert plugins.mock_calls == [
        call.initialize(),
        call.get_my_id(),
        # check if we have a db identifier set
        call.dcs_get_database_identifier(),
        # no, ok, init our db
        call.pg_initdb(),
        # make sure it starts
        call.pg_start(),
        call.pg_get_database_identifier(),
        # lock the database identifier so no-one else gets here
        call.dcs_lock('database_identifier')
    ]
    # shut down cleanly
    assert timeout == 5
Пример #11
0
def test_replica_bootstrap_fails_sanity_test(app):
    plugins = setup_plugins(app,
            pg_am_i_replica=False,
            dcs_get_database_identifier='1234',
            pg_get_database_identifier='42')
    timeout = app.initialize()
    assert app._plugins.mock_calls ==  [
            call.initialize(),
            call.get_my_id(),
            # compare our id with the id in the DCS
            call.dcs_get_database_identifier(),
            call.pg_get_database_identifier(),
            # make sure postgresql is stopped
            call.pg_stop(),
            # postgresql restore
            call.pg_restore(),
            call.pg_setup_replication(),
            call.pg_am_i_replica(),
            call.pg_reset(),
            ]
    # shut down after 5 seconds to try again
    assert timeout == 5
Пример #12
0
def test_replica_bootstrap_fails_sanity_test(app):
    plugins = setup_plugins(app,
                            pg_replication_role='master',
                            dcs_get_database_identifier='1234',
                            pg_get_database_identifier='42')
    timeout = app.initialize()
    assert plugins.mock_calls == [
        call.initialize(),
        call.get_my_id(),
        # compare our id with the id in the DCS
        call.dcs_get_database_identifier(),
        call.pg_get_database_identifier(),
        # make sure postgresql is stopped
        call.pg_stop(),
        # postgresql restore
        call.pg_initdb(),
        call.pg_restore(),
        call.pg_setup_replication(None),
        call.pg_get_database_identifier(),
        call.pg_replication_role(),
        call.pg_reset(),
    ]
    # shut down after 5 seconds to try again
    assert timeout == 5
Пример #13
0
def test_replica_start(app):
    plugins = setup_plugins(app,
                            dcs_get_database_identifier='1234',
                            dcs_lock=True,
                            pg_replication_role='replica',
                            pg_get_database_identifier='1234')
    app._conn_info['a'] = 'b'

    def start_monitoring():
        app.unhealthy('test_monitor', 'Waiting for first check')

    plugins.start_monitoring.side_effect = start_monitoring
    # sync startup
    timeout = app.initialize()
    assert plugins.mock_calls == [
        call.initialize(),
        call.get_my_id(),
        # compare our id with the id in the DCS
        call.dcs_get_database_identifier(),
        call.pg_get_database_identifier(),
        # check if I am a replica
        call.pg_replication_role(),
        # not master, so sure the DB is running
        call.pg_start(),
        # start monitoring
        call.start_monitoring(),
        call.dcs_watch(
            app.master_lock_changed,
            app._notify_state,
            app._notify_conn_info,
        ),
        # setup our connection info
        call.get_conn_info(),
        # set our first state
        call.dcs_set_state({
            'a': 'b',
            'host': '127.0.0.1',
            'replication_role': 'replica',
            'health_problems': {
                'test_monitor': {
                    'can_be_replica': False,
                    'reason': 'Waiting for first check'
                }
            },
        })
    ]
    # Carry on running afterwards
    assert timeout == None
    assert app.health_problems == {
        'test_monitor': {
            'can_be_replica': False,
            'reason': 'Waiting for first check'
        }
    }
    # Our test monitor becomes healthy
    plugins.reset_mock()
    with patch('time.time') as mock_time:
        app.healthy('test_monitor')
    assert plugins.mock_calls == [
        call.veto_takeover({
            'health_problems': {},
            'a': 'b',
            'replication_role': 'replica',
            'host': '127.0.0.1'
        }),
        call.dcs_set_state({
            'health_problems': {},
            'a': 'b',
            'replication_role': 'replica',
            'host': '127.0.0.1',
            'willing': mock_time(),
        }),
        call.pg_replication_role(),
        call.dcs_set_conn_info({
            'a': 'b',
            'host': '127.0.0.1'
        }),
    ]