def test_clean_force(topology_m4, m4rid):
"""Check that multiple tasks with a 'force' option work properly
:id: eb76a93d-8d1c-405e-9f25-6e8d5a781098
:setup: Replication setup with four masters
:steps:
1. Stop master 3
2. Add a bunch of updates to master 4
3. Disable replication on master 4
4. Start master 3
5. Remove agreements to master 4 from other masters
6. Run a cleanallruv task on master 1 with a 'force' option 'on'
7. Check that everything was cleaned
:expectedresults:
1. Master 3 should be stopped
2. Operation should be successful
3. Replication on master 4 should be disabled
4. Master 3 should be started
5. Agreements to master 4 should be removed
6. Operation should be successful
7. Everything should be cleaned
"""
log.info('Running test_clean_force...')
# Stop master 3, while we update master 4, so that 3 is behind the other masters
topology_m4.ms["master3"].stop()
# Add a bunch of updates to master 4
m4_add_users = AddUsers(topology_m4.ms["master4"], 1500)
m4_add_users.start()
m4_add_users.join()
# Start master 3, it should be out of sync with the other replicas...
topology_m4.ms["master3"].start()
# Remove the agreements from the other masters that point to master 4
remove_master4_agmts("test_clean_force", topology_m4)
# Run the task, use "force" because master 3 is not in sync with the other replicas
# in regards to the replica 4 RUV
log.info('test_clean: run the cleanAllRUV task...')
cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])
cruv_task.create(
properties={
'replica-id': m4rid,
'replica-base-dn': DEFAULT_SUFFIX,
'replica-force-cleaning': 'yes'
})
cruv_task.wait()
# Check the other master's RUV for 'replica 4'
log.info('test_clean_force: check all the masters have been cleaned...')
clean = check_ruvs("test_clean_force", topology_m4, m4rid)
assert clean
log.info('test_clean_force PASSED, restoring master 4...')
def fin():
try:
cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])
cruv_task.create(properties={
'replica-id': m4rid,
'replica-base-dn': DEFAULT_SUFFIX
})
cruv_task.wait()
except ldap.UNWILLING_TO_PERFORM:
# In some casse we already cleaned rid4, so if we fail, it's okay
pass
restore_master4(topology_m4)
# Make sure everything works.
repl.test_replication_topology(topology_m4.ms.values())
def fin():
try:
# Restart the suppliers and rerun cleanallruv
for inst in topology_m4.ms.values():
inst.restart()
cruv_task = CleanAllRUVTask(topology_m4.ms["supplier1"])
cruv_task.create(properties={
'replica-id': m4rid,
'replica-base-dn': DEFAULT_SUFFIX,
'replica-force-cleaning': 'no',
})
cruv_task.wait()
except ldap.UNWILLING_TO_PERFORM:
# In some casse we already cleaned rid4, so if we fail, it's okay
pass
restore_supplier4(topology_m4)
# Make sure everything works.
repl.test_replication_topology(topology_m4.ms.values())
def test_clean(topology_m4, m4rid):
"""Check that cleanallruv task works properly
:id: e9b3ce5c-e17c-409e-aafc-e97d630f2878
:setup: Replication setup with four masters
:steps:
1. Check that replication works on all masters
2. Disable replication on master 4
3. Remove agreements to master 4 from other masters
4. Run a cleanallruv task on master 1 with a 'force' option 'on'
5. Check that everything was cleaned
:expectedresults:
1. Replication should work properly on all masters
2. Operation should be successful
3. Agreements to master 4 should be removed
4. Cleanallruv task should be successfully executed
5. Everything should be cleaned
"""
log.info('Running test_clean...')
# Disable master 4
# Remove the agreements from the other masters that point to master 4
log.info('test_clean: disable master 4...')
remove_master4_agmts("test_clean", topology_m4)
# Run the task
log.info('test_clean: run the cleanAllRUV task...')
cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])
cruv_task.create(
properties={
'replica-id': m4rid,
'replica-base-dn': DEFAULT_SUFFIX,
'replica-force-cleaning': 'no'
})
cruv_task.wait()
# Check the other master's RUV for 'replica 4'
log.info('test_clean: check all the masters have been cleaned...')
clean = check_ruvs("test_clean", topology_m4, m4rid)
assert clean
log.info('test_clean PASSED, restoring master 4...')
def test_multiple_tasks_with_force(topology_m4, m4rid):
"""Check that multiple tasks with a 'force' option work properly
:id: eb76a93d-8d1c-405e-9f25-6e8d5a781098
:setup: Replication setup with four masters
:steps:
1. Stop master 3
2. Add a bunch of updates to master 4
3. Disable replication on master 4
4. Start master 3
5. Remove agreements to master 4 from other masters
6. Run a cleanallruv task on master 1 with a 'force' option 'on'
7. Run one more cleanallruv task on master 1 with a 'force' option 'off'
8. Check that everything was cleaned
:expectedresults:
1. Master 3 should be stopped
2. Operation should be successful
3. Replication on master 4 should be disabled
4. Master 3 should be started
5. Agreements to master 4 should be removed
6. Operation should be successful
7. Operation should be successful
8. Everything should be cleaned
"""
log.info('Running test_multiple_tasks_with_force...')
# Stop master 3, while we update master 4, so that 3 is behind the other masters
topology_m4.ms["master3"].stop()
# Add a bunch of updates to master 4
m4_add_users = AddUsers(topology_m4.ms["master4"], 1500)
m4_add_users.start()
m4_add_users.join()
# Start master 3, it should be out of sync with the other replicas...
topology_m4.ms["master3"].start()
# Disable master 4
# Remove the agreements from the other masters that point to master 4
remove_master4_agmts("test_multiple_tasks_with_force", topology_m4)
# Run the task, use "force" because master 3 is not in sync with the other replicas
# in regards to the replica 4 RUV
log.info(
'test_multiple_tasks_with_force: run the cleanAllRUV task with "force" on...'
)
cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])
cruv_task.create(
properties={
'replica-id': m4rid,
'replica-base-dn': DEFAULT_SUFFIX,
'replica-force-cleaning': 'yes',
'replica-certify-all': 'no'
})
log.info(
'test_multiple_tasks_with_force: run the cleanAllRUV task with "force" off...'
)
# NOTE: This must be try not py.test raises, because the above may or may
# not have completed yet ....
try:
cruv_task_fail = CleanAllRUVTask(topology_m4.ms["master1"])
cruv_task_fail.create(
properties={
'replica-id': m4rid,
'replica-base-dn': DEFAULT_SUFFIX,
'replica-force-cleaning': 'no',
'replica-certify-all': 'no'
})
cruv_task_fail.wait()
except ldap.UNWILLING_TO_PERFORM:
pass
# Wait for the force task ....
cruv_task.wait()
# Check the other master's RUV for 'replica 4'
log.info(
'test_multiple_tasks_with_force: check all the masters have been cleaned...'
)
clean = check_ruvs("test_clean_force", topology_m4, m4rid)
assert clean
# Check master 1 does not have the clean task running
log.info('test_abort: check master 1 no longer has a cleanAllRUV task...')
if not task_done(topology_m4, cruv_task.dn):
log.fatal('test_abort: CleanAllRUV task was not aborted')
assert False
def test_stress_clean(topology_m4, m4rid):
"""Put each server(m1 - m4) under a stress, and perform the entire clean process
:id: a8263cd6-f068-4357-86e0-e7c34504c8c5
:setup: Replication setup with four masters
:steps:
1. Add a bunch of updates to all masters
2. Put master 4 to read-only mode
3. Disable replication on master 4
5. Remove agreements to master 4 from other masters
6. Run a cleanallruv task on master 1
7. Check that everything was cleaned
:expectedresults:
1. Operation should be successful
2. Master 4 should be put to read-only mode
3. Replication on master 4 should be disabled
2. Agreements to master 4 should be removed
5. Agreements to master 4 should be removed
6. Operation should be successful
7. Everything should be cleaned
"""
log.info('Running test_stress_clean...')
log.info('test_stress_clean: put all the masters under load...')
ldbm_config = LDBMConfig(topology_m4.ms["master4"])
# Put all the masters under load
m1_add_users = AddUsers(topology_m4.ms["master1"], 2000)
m1_add_users.start()
m2_add_users = AddUsers(topology_m4.ms["master2"], 2000)
m2_add_users.start()
m3_add_users = AddUsers(topology_m4.ms["master3"], 2000)
m3_add_users.start()
m4_add_users = AddUsers(topology_m4.ms["master4"], 2000)
m4_add_users.start()
# Allow sometime to get replication flowing in all directions
log.info(
'test_stress_clean: allow some time for replication to get flowing...')
time.sleep(5)
# Put master 4 into read only mode
ldbm_config.set('nsslapd-readonly', 'on')
# We need to wait for master 4 to push its changes out
log.info(
'test_stress_clean: allow some time for master 4 to push changes out (60 seconds)...'
)
time.sleep(30)
# Remove the agreements from the other masters that point to master 4
remove_master4_agmts("test_stress_clean", topology_m4)
# Run the task
cruv_task = CleanAllRUVTask(topology_m4.ms["master1"])
cruv_task.create(
properties={
'replica-id': m4rid,
'replica-base-dn': DEFAULT_SUFFIX,
'replica-force-cleaning': 'no'
})
cruv_task.wait()
# Wait for the update to finish
log.info('test_stress_clean: wait for all the updates to finish...')
m1_add_users.join()
m2_add_users.join()
m3_add_users.join()
m4_add_users.join()
# Check the other master's RUV for 'replica 4'
log.info(
'test_stress_clean: check if all the replicas have been cleaned...')
clean = check_ruvs("test_stress_clean", topology_m4, m4rid)
assert clean
log.info('test_stress_clean: PASSED, restoring master 4...')
# Sleep for a bit to replication complete
log.info("Sleep for 120 seconds to allow replication to complete...")
repl = ReplicationManager(DEFAULT_SUFFIX)
repl.test_replication_topology([
topology_m4.ms["master1"],
topology_m4.ms["master2"],
topology_m4.ms["master3"],
],
timeout=120)
# Turn off readonly mode
ldbm_config.set('nsslapd-readonly', 'off')
