def test_clean_force(topology_m4, m4rid): """Check that multiple tasks with a 'force' option work properly :id: eb76a93d-8d1c-405e-9f25-6e8d5a781098 :setup: Replication setup with four masters :steps: 1. Stop master 3 2. Add a bunch of updates to master 4 3. Disable replication on master 4 4. Start master 3 5. Remove agreements to master 4 from other masters 6. Run a cleanallruv task on master 1 with a 'force' option 'on' 7. Check that everything was cleaned :expectedresults: 1. Master 3 should be stopped 2. Operation should be successful 3. Replication on master 4 should be disabled 4. Master 3 should be started 5. Agreements to master 4 should be removed 6. Operation should be successful 7. Everything should be cleaned """ log.info('Running test_clean_force...') # Stop master 3, while we update master 4, so that 3 is behind the other masters topology_m4.ms["master3"].stop() # Add a bunch of updates to master 4 m4_add_users = AddUsers(topology_m4.ms["master4"], 1500) m4_add_users.start() m4_add_users.join() # Start master 3, it should be out of sync with the other replicas... topology_m4.ms["master3"].start() # Remove the agreements from the other masters that point to master 4 remove_master4_agmts("test_clean_force", topology_m4) # Run the task, use "force" because master 3 is not in sync with the other replicas # in regards to the replica 4 RUV log.info('test_clean: run the cleanAllRUV task...') cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create( properties={ 'replica-id': m4rid, 'replica-base-dn': DEFAULT_SUFFIX, 'replica-force-cleaning': 'yes' }) cruv_task.wait() # Check the other master's RUV for 'replica 4' log.info('test_clean_force: check all the masters have been cleaned...') clean = check_ruvs("test_clean_force", topology_m4, m4rid) assert clean log.info('test_clean_force PASSED, restoring master 4...')
def fin(): try: cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create(properties={ 'replica-id': m4rid, 'replica-base-dn': DEFAULT_SUFFIX }) cruv_task.wait() except ldap.UNWILLING_TO_PERFORM: # In some casse we already cleaned rid4, so if we fail, it's okay pass restore_master4(topology_m4) # Make sure everything works. repl.test_replication_topology(topology_m4.ms.values())
def fin(): try: # Restart the suppliers and rerun cleanallruv for inst in topology_m4.ms.values(): inst.restart() cruv_task = CleanAllRUVTask(topology_m4.ms["supplier1"]) cruv_task.create(properties={ 'replica-id': m4rid, 'replica-base-dn': DEFAULT_SUFFIX, 'replica-force-cleaning': 'no', }) cruv_task.wait() except ldap.UNWILLING_TO_PERFORM: # In some casse we already cleaned rid4, so if we fail, it's okay pass restore_supplier4(topology_m4) # Make sure everything works. repl.test_replication_topology(topology_m4.ms.values())
def test_clean(topology_m4, m4rid): """Check that cleanallruv task works properly :id: e9b3ce5c-e17c-409e-aafc-e97d630f2878 :setup: Replication setup with four masters :steps: 1. Check that replication works on all masters 2. Disable replication on master 4 3. Remove agreements to master 4 from other masters 4. Run a cleanallruv task on master 1 with a 'force' option 'on' 5. Check that everything was cleaned :expectedresults: 1. Replication should work properly on all masters 2. Operation should be successful 3. Agreements to master 4 should be removed 4. Cleanallruv task should be successfully executed 5. Everything should be cleaned """ log.info('Running test_clean...') # Disable master 4 # Remove the agreements from the other masters that point to master 4 log.info('test_clean: disable master 4...') remove_master4_agmts("test_clean", topology_m4) # Run the task log.info('test_clean: run the cleanAllRUV task...') cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create( properties={ 'replica-id': m4rid, 'replica-base-dn': DEFAULT_SUFFIX, 'replica-force-cleaning': 'no' }) cruv_task.wait() # Check the other master's RUV for 'replica 4' log.info('test_clean: check all the masters have been cleaned...') clean = check_ruvs("test_clean", topology_m4, m4rid) assert clean log.info('test_clean PASSED, restoring master 4...')
def test_multiple_tasks_with_force(topology_m4, m4rid): """Check that multiple tasks with a 'force' option work properly :id: eb76a93d-8d1c-405e-9f25-6e8d5a781098 :setup: Replication setup with four masters :steps: 1. Stop master 3 2. Add a bunch of updates to master 4 3. Disable replication on master 4 4. Start master 3 5. Remove agreements to master 4 from other masters 6. Run a cleanallruv task on master 1 with a 'force' option 'on' 7. Run one more cleanallruv task on master 1 with a 'force' option 'off' 8. Check that everything was cleaned :expectedresults: 1. Master 3 should be stopped 2. Operation should be successful 3. Replication on master 4 should be disabled 4. Master 3 should be started 5. Agreements to master 4 should be removed 6. Operation should be successful 7. Operation should be successful 8. Everything should be cleaned """ log.info('Running test_multiple_tasks_with_force...') # Stop master 3, while we update master 4, so that 3 is behind the other masters topology_m4.ms["master3"].stop() # Add a bunch of updates to master 4 m4_add_users = AddUsers(topology_m4.ms["master4"], 1500) m4_add_users.start() m4_add_users.join() # Start master 3, it should be out of sync with the other replicas... topology_m4.ms["master3"].start() # Disable master 4 # Remove the agreements from the other masters that point to master 4 remove_master4_agmts("test_multiple_tasks_with_force", topology_m4) # Run the task, use "force" because master 3 is not in sync with the other replicas # in regards to the replica 4 RUV log.info( 'test_multiple_tasks_with_force: run the cleanAllRUV task with "force" on...' ) cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create( properties={ 'replica-id': m4rid, 'replica-base-dn': DEFAULT_SUFFIX, 'replica-force-cleaning': 'yes', 'replica-certify-all': 'no' }) log.info( 'test_multiple_tasks_with_force: run the cleanAllRUV task with "force" off...' ) # NOTE: This must be try not py.test raises, because the above may or may # not have completed yet .... try: cruv_task_fail = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task_fail.create( properties={ 'replica-id': m4rid, 'replica-base-dn': DEFAULT_SUFFIX, 'replica-force-cleaning': 'no', 'replica-certify-all': 'no' }) cruv_task_fail.wait() except ldap.UNWILLING_TO_PERFORM: pass # Wait for the force task .... cruv_task.wait() # Check the other master's RUV for 'replica 4' log.info( 'test_multiple_tasks_with_force: check all the masters have been cleaned...' ) clean = check_ruvs("test_clean_force", topology_m4, m4rid) assert clean # Check master 1 does not have the clean task running log.info('test_abort: check master 1 no longer has a cleanAllRUV task...') if not task_done(topology_m4, cruv_task.dn): log.fatal('test_abort: CleanAllRUV task was not aborted') assert False
def test_stress_clean(topology_m4, m4rid): """Put each server(m1 - m4) under a stress, and perform the entire clean process :id: a8263cd6-f068-4357-86e0-e7c34504c8c5 :setup: Replication setup with four masters :steps: 1. Add a bunch of updates to all masters 2. Put master 4 to read-only mode 3. Disable replication on master 4 5. Remove agreements to master 4 from other masters 6. Run a cleanallruv task on master 1 7. Check that everything was cleaned :expectedresults: 1. Operation should be successful 2. Master 4 should be put to read-only mode 3. Replication on master 4 should be disabled 2. Agreements to master 4 should be removed 5. Agreements to master 4 should be removed 6. Operation should be successful 7. Everything should be cleaned """ log.info('Running test_stress_clean...') log.info('test_stress_clean: put all the masters under load...') ldbm_config = LDBMConfig(topology_m4.ms["master4"]) # Put all the masters under load m1_add_users = AddUsers(topology_m4.ms["master1"], 2000) m1_add_users.start() m2_add_users = AddUsers(topology_m4.ms["master2"], 2000) m2_add_users.start() m3_add_users = AddUsers(topology_m4.ms["master3"], 2000) m3_add_users.start() m4_add_users = AddUsers(topology_m4.ms["master4"], 2000) m4_add_users.start() # Allow sometime to get replication flowing in all directions log.info( 'test_stress_clean: allow some time for replication to get flowing...') time.sleep(5) # Put master 4 into read only mode ldbm_config.set('nsslapd-readonly', 'on') # We need to wait for master 4 to push its changes out log.info( 'test_stress_clean: allow some time for master 4 to push changes out (60 seconds)...' ) time.sleep(30) # Remove the agreements from the other masters that point to master 4 remove_master4_agmts("test_stress_clean", topology_m4) # Run the task cruv_task = CleanAllRUVTask(topology_m4.ms["master1"]) cruv_task.create( properties={ 'replica-id': m4rid, 'replica-base-dn': DEFAULT_SUFFIX, 'replica-force-cleaning': 'no' }) cruv_task.wait() # Wait for the update to finish log.info('test_stress_clean: wait for all the updates to finish...') m1_add_users.join() m2_add_users.join() m3_add_users.join() m4_add_users.join() # Check the other master's RUV for 'replica 4' log.info( 'test_stress_clean: check if all the replicas have been cleaned...') clean = check_ruvs("test_stress_clean", topology_m4, m4rid) assert clean log.info('test_stress_clean: PASSED, restoring master 4...') # Sleep for a bit to replication complete log.info("Sleep for 120 seconds to allow replication to complete...") repl = ReplicationManager(DEFAULT_SUFFIX) repl.test_replication_topology([ topology_m4.ms["master1"], topology_m4.ms["master2"], topology_m4.ms["master3"], ], timeout=120) # Turn off readonly mode ldbm_config.set('nsslapd-readonly', 'off')