def restore_supplier4(topology_m4): """In our tests will always be removing supplier 4, so we need a common way to restore it for another test """ log.info('Restoring supplier 4...') # Enable replication on supplier 4 M4 = topology_m4.ms["supplier4"] M1 = topology_m4.ms["supplier1"] repl = ReplicationManager(SUFFIX) repl.join_supplier(M1, M4) repl.ensure_agreement(M4, M1) repl.ensure_agreement(M1, M4) # Test Replication is working for num in range(2, 5): if topology_m4.ms["supplier1"].testReplication(DEFAULT_SUFFIX, topology_m4.ms["supplier{}".format(num)]): log.info('Replication is working m1 -> m{}.'.format(num)) else: log.fatal('restore_supplier4: Replication is not working from m1 -> m{}.'.format(num)) assert False time.sleep(1) # Check replication is working from supplier 4 to supplier1... if topology_m4.ms["supplier4"].testReplication(DEFAULT_SUFFIX, topology_m4.ms["supplier1"]): log.info('Replication is working m4 -> m1.') else: log.fatal('restore_supplier4: Replication is not working from m4 -> 1.') assert False time.sleep(5) log.info('Supplier 4 has been successfully restored.')
def test_ticket48266_fractional(topology_m2, entries): ents = topology_m2.ms["supplier1"].agreement.list(suffix=SUFFIX) assert len(ents) == 1 mod = [(ldap.MOD_REPLACE, 'nsDS5ReplicatedAttributeList', [b'(objectclass=*) $ EXCLUDE telephonenumber']), (ldap.MOD_REPLACE, 'nsds5ReplicaStripAttrs', [b'modifiersname modifytimestamp'])] ents = topology_m2.ms["supplier1"].agreement.list(suffix=SUFFIX) assert len(ents) == 1 m1_m2_agmt = ents[0].dn topology_m2.ms["supplier1"].modify_s(ents[0].dn, mod) ents = topology_m2.ms["supplier2"].agreement.list(suffix=SUFFIX) assert len(ents) == 1 topology_m2.ms["supplier2"].modify_s(ents[0].dn, mod) topology_m2.ms["supplier1"].restart() topology_m2.ms["supplier2"].restart() repl = ReplicationManager(DEFAULT_SUFFIX) repl.ensure_agreement(topology_m2.ms["supplier1"], topology_m2.ms["supplier2"]) repl.test_replication(topology_m2.ms["supplier1"], topology_m2.ms["supplier2"])
def fractional_server_to_replica(server, replica): repl = ReplicationManager(DEFAULT_SUFFIX) repl.ensure_agreement(server, replica) replica_server = Replicas(server).get(DEFAULT_SUFFIX) agmt_server = replica_server.get_agreements().list()[0] agmt_server.replace_many( ('nsDS5ReplicatedAttributeListTotal', '(objectclass=*) $ EXCLUDE telephoneNumber'), ('nsDS5ReplicatedAttributeList', '(objectclass=*) $ EXCLUDE telephoneNumber'), ('nsds5ReplicaStripAttrs', 'modifiersname modifytimestamp'), )
def restore_supplier4(topology_m4): """In our tests will always be removing supplier 4, so we need a common way to restore it for another test """ # Restart the remaining suppliers to allow rid 4 to be reused. for inst in topology_m4.ms.values(): inst.restart() repl = ReplicationManager(DEFAULT_SUFFIX) repl.join_supplier(topology_m4.ms["supplier1"], topology_m4.ms["supplier4"]) # Add the 2,3 -> 4 agmt. repl.ensure_agreement(topology_m4.ms["supplier2"], topology_m4.ms["supplier4"]) repl.ensure_agreement(topology_m4.ms["supplier3"], topology_m4.ms["supplier4"]) # And in reverse ... repl.ensure_agreement(topology_m4.ms["supplier4"], topology_m4.ms["supplier2"]) repl.ensure_agreement(topology_m4.ms["supplier4"], topology_m4.ms["supplier3"]) log.info('Supplier 4 has been successfully restored.')
def test_lastupdate_attr_before_init(topo_nr): """Check that LastUpdate replica attributes show right values :id: bc8ce431-ff65-41f5-9331-605cbcaaa887 :customerscenario: True :setup: Replication setup with supplier and consumer instances without initialization :steps: 1. Check nsds5replicaLastUpdateStart value 2. Check nsds5replicaLastUpdateEnd value 3. Check nsds5replicaLastUpdateStatus value 4. Check nsds5replicaLastUpdateStatusJSON is parsable :expectedresults: 1. nsds5replicaLastUpdateStart should be equal to 0 2. nsds5replicaLastUpdateEnd should be equal to 0 3. nsds5replicaLastUpdateStatus should not be equal to "Replica acquired successfully: Incremental update started" 4. Success """ supplier = topo_nr.ins["standalone1"] consumer = topo_nr.ins["standalone2"] repl = ReplicationManager(DEFAULT_SUFFIX) repl.create_first_supplier(supplier) # Manually create an un-synced consumer. consumer_replicas = Replicas(consumer) consumer_replicas.create( properties={ 'cn': 'replica', 'nsDS5ReplicaRoot': DEFAULT_SUFFIX, 'nsDS5ReplicaId': '65535', 'nsDS5Flags': '0', 'nsDS5ReplicaType': '2', }) agmt = repl.ensure_agreement(supplier, consumer) with pytest.raises(Exception): repl.wait_for_replication(supplier, consumer, timeout=5) assert agmt.get_attr_val_utf8( 'nsds5replicaLastUpdateStart') == "19700101000000Z" assert agmt.get_attr_val_utf8( "nsds5replicaLastUpdateEnd") == "19700101000000Z" assert "replica acquired successfully" not in agmt.get_attr_val_utf8_l( "nsds5replicaLastUpdateStatus") # make sure the JSON attribute is parsable json_status = agmt.get_attr_val_utf8("nsds5replicaLastUpdateStatusJSON") if json_status is not None: json_obj = json.loads(json_status) log.debug("JSON status message: {}".format(json_obj))
def test_lastupdate_attr_before_init(topo_nr): """Check that LastUpdate replica attributes show right values :id: bc8ce431-ff65-41f5-9331-605cbcaaa887 :setup: Replication setup with master and consumer instances without initialization :steps: 1. Check nsds5replicaLastUpdateStart value 2. Check nsds5replicaLastUpdateEnd value 3. Check nsds5replicaLastUpdateStatus value :expectedresults: 1. nsds5replicaLastUpdateStart should be equal to 0 2. nsds5replicaLastUpdateEnd should be equal to 0 3. nsds5replicaLastUpdateStatus should not be equal to "0 Replica acquired successfully: Incremental update started" """ master = topo_nr.ins["standalone1"] consumer = topo_nr.ins["standalone2"] repl = ReplicationManager(DEFAULT_SUFFIX) repl.create_first_master(master) # Manually create an un-synced consumer. consumer_replicas = Replicas(consumer) consumer_replicas.create( properties={ 'cn': 'replica', 'nsDS5ReplicaRoot': DEFAULT_SUFFIX, 'nsDS5ReplicaId': '65535', 'nsDS5Flags': '0', 'nsDS5ReplicaType': '2', }) agmt = repl.ensure_agreement(master, consumer) with pytest.raises(Exception): repl.wait_for_replication(master, consumer, timeout=5) assert agmt.get_attr_val_bytes( 'nsds5replicaLastUpdateStart') == b"19700101000000Z" assert agmt.get_attr_val_bytes( "nsds5replicaLastUpdateEnd") == b"19700101000000Z" assert b"Replica acquired successfully" not in agmt.get_attr_val_bytes( "nsds5replicaLastUpdateStatus")
def replicate_backend(s1, s2, beSuffix): repl = ReplicationManager(beSuffix) repl.create_first_master(s1) repl.join_master(s1, s2) repl.ensure_agreement(s1, s2) repl.ensure_agreement(s2, s2)
def create_topology(topo_dict, suffix=DEFAULT_SUFFIX): """Create a requested topology. Cascading replication scenario isn't supported :param topo_dict: a dictionary {ReplicaRole.STANDALONE: num, ReplicaRole.MASTER: num, ReplicaRole.CONSUMER: num} :type topo_dict: dict :param suffix: a suffix for the replication :type suffix: str :return - TopologyMain object """ if not topo_dict: ValueError("You need to specify the dict. For instance: {ReplicaRole.STANDALONE: 1}") if ReplicaRole.HUB in topo_dict.keys(): NotImplementedError("Cascading replication scenario isn't supported." "Please, use existing topology or create your own.") topo = _create_instances(topo_dict, suffix) # Start with a single master, and create it "first". first_master = None try: first_master = list(topo.ms.values())[0] log.info("Creating replication topology.") # Now get the first master ready. repl = ReplicationManager(DEFAULT_SUFFIX) repl.create_first_master(first_master) except IndexError: pass # Now init the other masters from this. # This will reinit m, and put a bi-directional agreement # in place. for m in topo.ms.values(): # Skip firstmaster. if m is first_master: continue log.info("Joining master %s to %s ..." % (m.serverid, first_master.serverid)) repl.join_master(first_master, m) # Mesh the master agreements. for mo in topo.ms.values(): for mi in topo.ms.values(): if mo is mi: continue log.info("Ensuring master %s to %s ..." % (mo.serverid, mi.serverid)) repl.ensure_agreement(mo, mi) # Add master -> consumer agreements. for c in topo.cs.values(): log.info("Joining consumer %s from %s ..." % (c.serverid, first_master.serverid)) repl.join_consumer(first_master, c) for m in topo.ms.values(): for c in topo.cs.values(): log.info("Ensuring consumer %s from %s ..." % (c.serverid, m.serverid)) repl.ensure_agreement(m, c) # Clear out the tmp dir for instance in topo: instance.clearTmpDir(__file__) return topo
def test_ticket47819(topology_st): """ from lib389.utils import * # Skip on older versions pytestmark = pytest.mark.skipif(ds_is_older('1.3.4'), reason="Not implemented") Testing precise tombstone purging: [1] Make sure "nsTombstoneCSN" is added to new tombstones [2] Make sure an import of a replication ldif adds "nsTombstoneCSN" to old tombstones [4] Test fixup task [3] Make sure tombstone purging works """ log.info('Testing Ticket 47819 - Test precise tombstone purging') # # Setup Replication # master = topology_st.standalone repl = ReplicationManager(DEFAULT_SUFFIX) repl.create_first_master(master) repl.ensure_agreement(master, master) # # Part 1 create a tombstone entry and make sure nsTombstoneCSN is added # log.info('Part 1: Add and then delete an entry to create a tombstone...') try: topology_st.standalone.add_s( Entry(('cn=entry1,dc=example,dc=com', { 'objectclass': 'top person'.split(), 'sn': 'user', 'cn': 'entry1' }))) except ldap.LDAPError as e: log.error('Failed to add entry: ' + e.message['desc']) assert False try: topology_st.standalone.delete_s('cn=entry1,dc=example,dc=com') except ldap.LDAPError as e: log.error('Failed to delete entry: ' + e.message['desc']) assert False log.info('Search for tombstone entries...') try: entries = topology_st.standalone.search_s( DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, '(&(nsTombstoneCSN=*)(objectclass=nsTombstone))') if not entries: log.fatal( 'Search failed to the new tombstone(nsTombstoneCSN is probably missing).' ) assert False except ldap.LDAPError as e: log.fatal('Search failed: ' + e.message['desc']) assert False log.info('Part 1 - passed') # # Part 2 - import ldif with tombstones missing 'nsTombstoneCSN' # # First, export the replication ldif, edit the file(remove nstombstonecsn), # and reimport it. # log.info('Part 2: Exporting replication ldif...') # Get the the full path and name for our LDIF we will be exporting ldif_file = "/tmp/export.ldif" args = {EXPORT_REPL_INFO: True, TASK_WAIT: True} exportTask = Tasks(topology_st.standalone) try: exportTask.exportLDIF(DEFAULT_SUFFIX, None, ldif_file, args) except ValueError: assert False time.sleep(1) # open the ldif file, get the lines, then rewrite the file ldif = open(ldif_file, "r") lines = ldif.readlines() ldif.close() time.sleep(1) ldif = open(ldif_file, "w") for line in lines: if not line.lower().startswith('nstombstonecsn'): ldif.write(line) ldif.close() time.sleep(1) # import the new ldif file log.info('Import replication LDIF file...') importTask = Tasks(topology_st.standalone) args = {TASK_WAIT: True} try: importTask.importLDIF(DEFAULT_SUFFIX, None, ldif_file, args) os.remove(ldif_file) except ValueError: os.remove(ldif_file) assert False time.sleep(1) # Search for the tombstone again log.info('Search for tombstone entries...') try: entries = topology_st.standalone.search_s( DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, '(&(nsTombstoneCSN=*)(objectclass=nsTombstone))') if not entries: log.fatal( 'Search failed to fine the new tombstone(nsTombstoneCSN is probably missing).' ) assert False except ldap.LDAPError as e: log.fatal('Search failed: ' + e.message['desc']) assert False log.info('Part 2 - passed') # # Part 3 - test fixup task # log.info('Part 3: test the fixup task') # Run fixup task using the strip option. This removes nsTombstoneCSN # so we can test if the fixup task works. args = {TASK_WAIT: True, TASK_TOMB_STRIP: True} fixupTombTask = Tasks(topology_st.standalone) try: fixupTombTask.fixupTombstones(DEFAULT_BENAME, args) except: assert False time.sleep(1) # Search for tombstones with nsTombstoneCSN - better not find any log.info('Search for tombstone entries...') try: entries = topology_st.standalone.search_s( DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, '(&(nsTombstoneCSN=*)(objectclass=nsTombstone))') if entries: log.fatal('Search found tombstones with nsTombstoneCSN') assert False except ldap.LDAPError as e: log.fatal('Search failed: ' + e.message['desc']) assert False # Now run the fixup task args = {TASK_WAIT: True} fixupTombTask = Tasks(topology_st.standalone) try: fixupTombTask.fixupTombstones(DEFAULT_BENAME, args) except: assert False time.sleep(1) # Search for tombstones with nsTombstoneCSN - better find some log.info('Search for tombstone entries...') try: entries = topology_st.standalone.search_s( DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, '(&(nsTombstoneCSN=*)(objectclass=nsTombstone))') if not entries: log.fatal('Search did not find any fixed-up tombstones') assert False except ldap.LDAPError as e: log.fatal('Search failed: ' + e.message['desc']) assert False log.info('Part 3 - passed') # # Part 4 - Test tombstone purging # log.info('Part 4: test tombstone purging...') args = { REPLICA_PRECISE_PURGING: b'on', REPLICA_PURGE_DELAY: b'5', REPLICA_PURGE_INTERVAL: b'5' } try: topology_st.standalone.replica.setProperties(DEFAULT_SUFFIX, None, None, args) except: log.fatal('Failed to configure replica') assert False # Wait for the interval to pass log.info('Wait for tombstone purge interval to pass...') time.sleep(10) # Add an entry to trigger replication log.info('Perform an update to help trigger tombstone purging...') try: topology_st.standalone.add_s( Entry(('cn=test_entry,dc=example,dc=com', { 'objectclass': 'top person'.split(), 'sn': 'user', 'cn': 'entry1' }))) except ldap.LDAPError as e: log.error('Failed to add entry: ' + e.message['desc']) assert False # Wait for the interval to pass again log.info('Wait for tombstone purge interval to pass again...') time.sleep(10) # search for tombstones, there should be none log.info('Search for tombstone entries...') try: entries = topology_st.standalone.search_s( DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE, '(&(nsTombstoneCSN=*)(objectclass=nsTombstone))') if entries: log.fatal('Search unexpectedly found tombstones') assert False except ldap.LDAPError as e: log.fatal('Search failed: ' + e.message['desc']) assert False log.info('Part 4 - passed')