def restore_supplier4(topology_m4):
"""In our tests will always be removing supplier 4, so we need a common
way to restore it for another test
"""
log.info('Restoring supplier 4...')
# Enable replication on supplier 4
M4 = topology_m4.ms["supplier4"]
M1 = topology_m4.ms["supplier1"]
repl = ReplicationManager(SUFFIX)
repl.join_supplier(M1, M4)
repl.ensure_agreement(M4, M1)
repl.ensure_agreement(M1, M4)
# Test Replication is working
for num in range(2, 5):
if topology_m4.ms["supplier1"].testReplication(DEFAULT_SUFFIX, topology_m4.ms["supplier{}".format(num)]):
log.info('Replication is working m1 -> m{}.'.format(num))
else:
log.fatal('restore_supplier4: Replication is not working from m1 -> m{}.'.format(num))
assert False
time.sleep(1)
# Check replication is working from supplier 4 to supplier1...
if topology_m4.ms["supplier4"].testReplication(DEFAULT_SUFFIX, topology_m4.ms["supplier1"]):
log.info('Replication is working m4 -> m1.')
else:
log.fatal('restore_supplier4: Replication is not working from m4 -> 1.')
assert False
time.sleep(5)
log.info('Supplier 4 has been successfully restored.')
def test_ticket48266_fractional(topology_m2, entries):
ents = topology_m2.ms["supplier1"].agreement.list(suffix=SUFFIX)
assert len(ents) == 1
mod = [(ldap.MOD_REPLACE, 'nsDS5ReplicatedAttributeList',
[b'(objectclass=*) $ EXCLUDE telephonenumber']),
(ldap.MOD_REPLACE, 'nsds5ReplicaStripAttrs',
[b'modifiersname modifytimestamp'])]
ents = topology_m2.ms["supplier1"].agreement.list(suffix=SUFFIX)
assert len(ents) == 1
m1_m2_agmt = ents[0].dn
topology_m2.ms["supplier1"].modify_s(ents[0].dn, mod)
ents = topology_m2.ms["supplier2"].agreement.list(suffix=SUFFIX)
assert len(ents) == 1
topology_m2.ms["supplier2"].modify_s(ents[0].dn, mod)
topology_m2.ms["supplier1"].restart()
topology_m2.ms["supplier2"].restart()
repl = ReplicationManager(DEFAULT_SUFFIX)
repl.ensure_agreement(topology_m2.ms["supplier1"],
topology_m2.ms["supplier2"])
repl.test_replication(topology_m2.ms["supplier1"],
topology_m2.ms["supplier2"])
def fractional_server_to_replica(server, replica):
repl = ReplicationManager(DEFAULT_SUFFIX)
repl.ensure_agreement(server, replica)
replica_server = Replicas(server).get(DEFAULT_SUFFIX)
agmt_server = replica_server.get_agreements().list()[0]
agmt_server.replace_many(
('nsDS5ReplicatedAttributeListTotal', '(objectclass=*) $ EXCLUDE telephoneNumber'),
('nsDS5ReplicatedAttributeList', '(objectclass=*) $ EXCLUDE telephoneNumber'),
('nsds5ReplicaStripAttrs', 'modifiersname modifytimestamp'),
)
def restore_supplier4(topology_m4):
"""In our tests will always be removing supplier 4, so we need a common
way to restore it for another test
"""
# Restart the remaining suppliers to allow rid 4 to be reused.
for inst in topology_m4.ms.values():
inst.restart()
repl = ReplicationManager(DEFAULT_SUFFIX)
repl.join_supplier(topology_m4.ms["supplier1"],
topology_m4.ms["supplier4"])
# Add the 2,3 -> 4 agmt.
repl.ensure_agreement(topology_m4.ms["supplier2"],
topology_m4.ms["supplier4"])
repl.ensure_agreement(topology_m4.ms["supplier3"],
topology_m4.ms["supplier4"])
# And in reverse ...
repl.ensure_agreement(topology_m4.ms["supplier4"],
topology_m4.ms["supplier2"])
repl.ensure_agreement(topology_m4.ms["supplier4"],
topology_m4.ms["supplier3"])
log.info('Supplier 4 has been successfully restored.')
def test_lastupdate_attr_before_init(topo_nr):
"""Check that LastUpdate replica attributes show right values
:id: bc8ce431-ff65-41f5-9331-605cbcaaa887
:customerscenario: True
:setup: Replication setup with supplier and consumer instances
without initialization
:steps:
1. Check nsds5replicaLastUpdateStart value
2. Check nsds5replicaLastUpdateEnd value
3. Check nsds5replicaLastUpdateStatus value
4. Check nsds5replicaLastUpdateStatusJSON is parsable
:expectedresults:
1. nsds5replicaLastUpdateStart should be equal to 0
2. nsds5replicaLastUpdateEnd should be equal to 0
3. nsds5replicaLastUpdateStatus should not be equal
to "Replica acquired successfully: Incremental update started"
4. Success
"""
supplier = topo_nr.ins["standalone1"]
consumer = topo_nr.ins["standalone2"]
repl = ReplicationManager(DEFAULT_SUFFIX)
repl.create_first_supplier(supplier)
# Manually create an un-synced consumer.
consumer_replicas = Replicas(consumer)
consumer_replicas.create(
properties={
'cn': 'replica',
'nsDS5ReplicaRoot': DEFAULT_SUFFIX,
'nsDS5ReplicaId': '65535',
'nsDS5Flags': '0',
'nsDS5ReplicaType': '2',
})
agmt = repl.ensure_agreement(supplier, consumer)
with pytest.raises(Exception):
repl.wait_for_replication(supplier, consumer, timeout=5)
assert agmt.get_attr_val_utf8(
'nsds5replicaLastUpdateStart') == "19700101000000Z"
assert agmt.get_attr_val_utf8(
"nsds5replicaLastUpdateEnd") == "19700101000000Z"
assert "replica acquired successfully" not in agmt.get_attr_val_utf8_l(
"nsds5replicaLastUpdateStatus")
# make sure the JSON attribute is parsable
json_status = agmt.get_attr_val_utf8("nsds5replicaLastUpdateStatusJSON")
if json_status is not None:
json_obj = json.loads(json_status)
log.debug("JSON status message: {}".format(json_obj))
def test_lastupdate_attr_before_init(topo_nr):
"""Check that LastUpdate replica attributes show right values
:id: bc8ce431-ff65-41f5-9331-605cbcaaa887
:setup: Replication setup with master and consumer instances
without initialization
:steps:
1. Check nsds5replicaLastUpdateStart value
2. Check nsds5replicaLastUpdateEnd value
3. Check nsds5replicaLastUpdateStatus value
:expectedresults:
1. nsds5replicaLastUpdateStart should be equal to 0
2. nsds5replicaLastUpdateEnd should be equal to 0
3. nsds5replicaLastUpdateStatus should not be equal
to "0 Replica acquired successfully: Incremental update started"
"""
master = topo_nr.ins["standalone1"]
consumer = topo_nr.ins["standalone2"]
repl = ReplicationManager(DEFAULT_SUFFIX)
repl.create_first_master(master)
# Manually create an un-synced consumer.
consumer_replicas = Replicas(consumer)
consumer_replicas.create(
properties={
'cn': 'replica',
'nsDS5ReplicaRoot': DEFAULT_SUFFIX,
'nsDS5ReplicaId': '65535',
'nsDS5Flags': '0',
'nsDS5ReplicaType': '2',
})
agmt = repl.ensure_agreement(master, consumer)
with pytest.raises(Exception):
repl.wait_for_replication(master, consumer, timeout=5)
assert agmt.get_attr_val_bytes(
'nsds5replicaLastUpdateStart') == b"19700101000000Z"
assert agmt.get_attr_val_bytes(
"nsds5replicaLastUpdateEnd") == b"19700101000000Z"
assert b"Replica acquired successfully" not in agmt.get_attr_val_bytes(
"nsds5replicaLastUpdateStatus")
def replicate_backend(s1, s2, beSuffix):
repl = ReplicationManager(beSuffix)
repl.create_first_master(s1)
repl.join_master(s1, s2)
repl.ensure_agreement(s1, s2)
repl.ensure_agreement(s2, s2)
def create_topology(topo_dict, suffix=DEFAULT_SUFFIX):
"""Create a requested topology. Cascading replication scenario isn't supported
:param topo_dict: a dictionary {ReplicaRole.STANDALONE: num, ReplicaRole.MASTER: num,
ReplicaRole.CONSUMER: num}
:type topo_dict: dict
:param suffix: a suffix for the replication
:type suffix: str
:return - TopologyMain object
"""
if not topo_dict:
ValueError("You need to specify the dict. For instance: {ReplicaRole.STANDALONE: 1}")
if ReplicaRole.HUB in topo_dict.keys():
NotImplementedError("Cascading replication scenario isn't supported."
"Please, use existing topology or create your own.")
topo = _create_instances(topo_dict, suffix)
# Start with a single master, and create it "first".
first_master = None
try:
first_master = list(topo.ms.values())[0]
log.info("Creating replication topology.")
# Now get the first master ready.
repl = ReplicationManager(DEFAULT_SUFFIX)
repl.create_first_master(first_master)
except IndexError:
pass
# Now init the other masters from this.
# This will reinit m, and put a bi-directional agreement
# in place.
for m in topo.ms.values():
# Skip firstmaster.
if m is first_master:
continue
log.info("Joining master %s to %s ..." % (m.serverid, first_master.serverid))
repl.join_master(first_master, m)
# Mesh the master agreements.
for mo in topo.ms.values():
for mi in topo.ms.values():
if mo is mi:
continue
log.info("Ensuring master %s to %s ..." % (mo.serverid, mi.serverid))
repl.ensure_agreement(mo, mi)
# Add master -> consumer agreements.
for c in topo.cs.values():
log.info("Joining consumer %s from %s ..." % (c.serverid, first_master.serverid))
repl.join_consumer(first_master, c)
for m in topo.ms.values():
for c in topo.cs.values():
log.info("Ensuring consumer %s from %s ..." % (c.serverid, m.serverid))
repl.ensure_agreement(m, c)
# Clear out the tmp dir
for instance in topo:
instance.clearTmpDir(__file__)
return topo
def test_ticket47819(topology_st):
"""
from lib389.utils import *
# Skip on older versions
pytestmark = pytest.mark.skipif(ds_is_older('1.3.4'), reason="Not implemented")
Testing precise tombstone purging:
[1] Make sure "nsTombstoneCSN" is added to new tombstones
[2] Make sure an import of a replication ldif adds "nsTombstoneCSN"
to old tombstones
[4] Test fixup task
[3] Make sure tombstone purging works
"""
log.info('Testing Ticket 47819 - Test precise tombstone purging')
#
# Setup Replication
#
master = topology_st.standalone
repl = ReplicationManager(DEFAULT_SUFFIX)
repl.create_first_master(master)
repl.ensure_agreement(master, master)
#
# Part 1 create a tombstone entry and make sure nsTombstoneCSN is added
#
log.info('Part 1: Add and then delete an entry to create a tombstone...')
try:
topology_st.standalone.add_s(
Entry(('cn=entry1,dc=example,dc=com', {
'objectclass': 'top person'.split(),
'sn': 'user',
'cn': 'entry1'
})))
except ldap.LDAPError as e:
log.error('Failed to add entry: ' + e.message['desc'])
assert False
try:
topology_st.standalone.delete_s('cn=entry1,dc=example,dc=com')
except ldap.LDAPError as e:
log.error('Failed to delete entry: ' + e.message['desc'])
assert False
log.info('Search for tombstone entries...')
try:
entries = topology_st.standalone.search_s(
DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE,
'(&(nsTombstoneCSN=*)(objectclass=nsTombstone))')
if not entries:
log.fatal(
'Search failed to the new tombstone(nsTombstoneCSN is probably missing).'
)
assert False
except ldap.LDAPError as e:
log.fatal('Search failed: ' + e.message['desc'])
assert False
log.info('Part 1 - passed')
#
# Part 2 - import ldif with tombstones missing 'nsTombstoneCSN'
#
# First, export the replication ldif, edit the file(remove nstombstonecsn),
# and reimport it.
#
log.info('Part 2: Exporting replication ldif...')
# Get the the full path and name for our LDIF we will be exporting
ldif_file = "/tmp/export.ldif"
args = {EXPORT_REPL_INFO: True, TASK_WAIT: True}
exportTask = Tasks(topology_st.standalone)
try:
exportTask.exportLDIF(DEFAULT_SUFFIX, None, ldif_file, args)
except ValueError:
assert False
time.sleep(1)
# open the ldif file, get the lines, then rewrite the file
ldif = open(ldif_file, "r")
lines = ldif.readlines()
ldif.close()
time.sleep(1)
ldif = open(ldif_file, "w")
for line in lines:
if not line.lower().startswith('nstombstonecsn'):
ldif.write(line)
ldif.close()
time.sleep(1)
# import the new ldif file
log.info('Import replication LDIF file...')
importTask = Tasks(topology_st.standalone)
args = {TASK_WAIT: True}
try:
importTask.importLDIF(DEFAULT_SUFFIX, None, ldif_file, args)
os.remove(ldif_file)
except ValueError:
os.remove(ldif_file)
assert False
time.sleep(1)
# Search for the tombstone again
log.info('Search for tombstone entries...')
try:
entries = topology_st.standalone.search_s(
DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE,
'(&(nsTombstoneCSN=*)(objectclass=nsTombstone))')
if not entries:
log.fatal(
'Search failed to fine the new tombstone(nsTombstoneCSN is probably missing).'
)
assert False
except ldap.LDAPError as e:
log.fatal('Search failed: ' + e.message['desc'])
assert False
log.info('Part 2 - passed')
#
# Part 3 - test fixup task
#
log.info('Part 3: test the fixup task')
# Run fixup task using the strip option. This removes nsTombstoneCSN
# so we can test if the fixup task works.
args = {TASK_WAIT: True, TASK_TOMB_STRIP: True}
fixupTombTask = Tasks(topology_st.standalone)
try:
fixupTombTask.fixupTombstones(DEFAULT_BENAME, args)
except:
assert False
time.sleep(1)
# Search for tombstones with nsTombstoneCSN - better not find any
log.info('Search for tombstone entries...')
try:
entries = topology_st.standalone.search_s(
DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE,
'(&(nsTombstoneCSN=*)(objectclass=nsTombstone))')
if entries:
log.fatal('Search found tombstones with nsTombstoneCSN')
assert False
except ldap.LDAPError as e:
log.fatal('Search failed: ' + e.message['desc'])
assert False
# Now run the fixup task
args = {TASK_WAIT: True}
fixupTombTask = Tasks(topology_st.standalone)
try:
fixupTombTask.fixupTombstones(DEFAULT_BENAME, args)
except:
assert False
time.sleep(1)
# Search for tombstones with nsTombstoneCSN - better find some
log.info('Search for tombstone entries...')
try:
entries = topology_st.standalone.search_s(
DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE,
'(&(nsTombstoneCSN=*)(objectclass=nsTombstone))')
if not entries:
log.fatal('Search did not find any fixed-up tombstones')
assert False
except ldap.LDAPError as e:
log.fatal('Search failed: ' + e.message['desc'])
assert False
log.info('Part 3 - passed')
#
# Part 4 - Test tombstone purging
#
log.info('Part 4: test tombstone purging...')
args = {
REPLICA_PRECISE_PURGING: b'on',
REPLICA_PURGE_DELAY: b'5',
REPLICA_PURGE_INTERVAL: b'5'
}
try:
topology_st.standalone.replica.setProperties(DEFAULT_SUFFIX, None,
None, args)
except:
log.fatal('Failed to configure replica')
assert False
# Wait for the interval to pass
log.info('Wait for tombstone purge interval to pass...')
time.sleep(10)
# Add an entry to trigger replication
log.info('Perform an update to help trigger tombstone purging...')
try:
topology_st.standalone.add_s(
Entry(('cn=test_entry,dc=example,dc=com', {
'objectclass': 'top person'.split(),
'sn': 'user',
'cn': 'entry1'
})))
except ldap.LDAPError as e:
log.error('Failed to add entry: ' + e.message['desc'])
assert False
# Wait for the interval to pass again
log.info('Wait for tombstone purge interval to pass again...')
time.sleep(10)
# search for tombstones, there should be none
log.info('Search for tombstone entries...')
try:
entries = topology_st.standalone.search_s(
DEFAULT_SUFFIX, ldap.SCOPE_SUBTREE,
'(&(nsTombstoneCSN=*)(objectclass=nsTombstone))')
if entries:
log.fatal('Search unexpectedly found tombstones')
assert False
except ldap.LDAPError as e:
log.fatal('Search failed: ' + e.message['desc'])
assert False
log.info('Part 4 - passed')