def measureConvergence(topology):
"""Find and measure the convergence of entries from each master
"""
replicas = [
topology.master1, topology.master2, topology.master3, topology.master4,
topology.hub1, topology.hub2, topology.consumer1, topology.consumer2,
topology.consumer3, topology.consumer4
]
if ADD_DEL_COUNT > 10:
interval = int(ADD_DEL_COUNT / 10)
else:
interval = 1
for master in [('1', topology.master1), ('2', topology.master2),
('3', topology.master3), ('4', topology.master4)]:
# Start with the first entry
entries = ['ADD dn="uid=master_%s-0,%s' % (master[0], DEFAULT_SUFFIX)]
# Add incremental entries to the list
idx = interval
while idx < ADD_DEL_COUNT:
entries.append('ADD dn="uid=master_%s-%d,%s' %
(master[0], idx, DEFAULT_SUFFIX))
idx += interval
# Add the last entry to the list (if it was not already added)
if idx != (ADD_DEL_COUNT - 1):
entries.append('ADD dn="uid=master_%s-%d,%s' %
(master[0], (ADD_DEL_COUNT - 1), DEFAULT_SUFFIX))
ReplTools.replConvReport(DEFAULT_SUFFIX, entries, master[1], replicas)
def measureConvergence(topology):
"""Find and measure the convergence of entries from each master
"""
replicas = [topology.master1, topology.master2, topology.master3,
topology.master4]
if ADD_DEL_COUNT > 10:
interval = int(ADD_DEL_COUNT / 10)
else:
interval = 1
for master in [('1', topology.master1),
('2', topology.master2),
('3', topology.master3),
('4', topology.master4)]:
# Start with the first entry
entries = ['ADD dn="uid=master_%s-0,%s' %
(master[0], DEFAULT_SUFFIX)]
# Add incremental entries to the list
idx = interval
while idx < ADD_DEL_COUNT:
entries.append('ADD dn="uid=master_%s-%d,%s' %
(master[0], idx, DEFAULT_SUFFIX))
idx += interval
# Add the last entry to the list (if it was not already added)
if idx != (ADD_DEL_COUNT - 1):
entries.append('ADD dn="uid=master_%s-%d,%s' %
(master[0], (ADD_DEL_COUNT - 1),
DEFAULT_SUFFIX))
ReplTools.replConvReport(DEFAULT_SUFFIX, entries, master[1], replicas)
def test_MMR_Integrity(topology):
"""Apply load to 4 masters at the same time. Perform adds and deletes.
If any updates are missed we will see an error 32 in the access logs or
we will have entries left over once the test completes.
"""
loop = 0
ALL_REPLICAS = [
topology.master1, topology.master2, topology.master3, topology.master4,
topology.hub1, topology.hub2, topology.consumer1, topology.consumer2,
topology.consumer3, topology.consumer4
]
if TEST_CONVERGE_LATENCY:
try:
for inst in ALL_REPLICAS:
replica = inst.replicas.get(DEFAULT_SUFFIX)
replica.set('nsds5ReplicaReleaseTimeout', CONVERGENCE_TIMEOUT)
except ldap.LDAPError as e:
log.fatal('Failed to set replicas release timeout - error: %s' %
(str(e)))
assert False
if DEBUGGING:
# Enable Repl logging, and increase the max logs
try:
for inst in ALL_REPLICAS:
inst.enableReplLogging()
inst.modify_s("cn=config", [
(ldap.MOD_REPLACE, 'nsslapd-errorlog-maxlogsperdir', '5')
])
except ldap.LDAPError as e:
log.fatal('Failed to set max logs - error: %s' % (str(e)))
assert False
while loop < MAX_LOOPS:
# Remove the current logs so we have a clean set of logs to check.
log.info('Pass %d...' % (loop + 1))
log.info("Removing logs...")
for inst in ALL_REPLICAS:
inst.deleteAllLogs()
# Fire off 4 threads to apply the load
log.info("Start adding/deleting: " + getDateTime())
startTime = time.time()
add_del_m1 = AddDelUsers(topology.master1)
add_del_m1.start()
add_del_m2 = AddDelUsers(topology.master2)
add_del_m2.start()
add_del_m3 = AddDelUsers(topology.master3)
add_del_m3.start()
add_del_m4 = AddDelUsers(topology.master4)
add_del_m4.start()
# Wait for threads to finish sending their updates
add_del_m1.join()
add_del_m2.join()
add_del_m3.join()
add_del_m4.join()
log.info("Finished adding/deleting entries: " + getDateTime())
#
# Loop checking for error 32's, and for convergence to complete
#
log.info("Waiting for replication to converge...")
while True:
# First check for error 32's
for inst in ALL_REPLICAS:
if inst.searchAccessLog(" err=32 "):
log.fatal('An add was missed on: ' + inst.serverid)
assert False
# Next check to see if the last update is in the access log
converged = True
for inst in ALL_REPLICAS:
if not inst.searchAccessLog(LAST_DN_M1) or \
not inst.searchAccessLog(LAST_DN_M2) or \
not inst.searchAccessLog(LAST_DN_M3) or \
not inst.searchAccessLog(LAST_DN_M4):
converged = False
break
if converged:
elapsed_tm = int(time.time() - startTime)
convtime = str(datetime.timedelta(seconds=elapsed_tm))
log.info('Replication converged at: ' + getDateTime() +
' - Elapsed Time: ' + convtime)
break
else:
# Check if replication is idle
replicas = [
topology.master1, topology.master2, topology.master3,
topology.master4, topology.hub1, topology.hub2
]
if ReplTools.replIdle(replicas, DEFAULT_SUFFIX):
# Replication is idle - wait 30 secs for access log buffer
time.sleep(30)
# Now check the access log again...
converged = True
for inst in ALL_REPLICAS:
if not inst.searchAccessLog(LAST_DN_M1) or \
not inst.searchAccessLog(LAST_DN_M2) or \
not inst.searchAccessLog(LAST_DN_M3) or \
not inst.searchAccessLog(LAST_DN_M4):
converged = False
break
if converged:
elapsed_tm = int(time.time() - startTime)
convtime = str(datetime.timedelta(seconds=elapsed_tm))
log.info('Replication converged at: ' + getDateTime() +
' - Elapsed Time: ' + convtime)
break
else:
log.fatal('Stopping replication check: ' +
getDateTime())
log.fatal('Failure: Replication is complete, but we ' +
'never converged.')
assert False
# Sleep a bit before the next pass
time.sleep(3)
#
# Finally check the CSN's
#
log.info("Check the CSN's...")
if not ReplTools.checkCSNs(ALL_REPLICAS):
assert False
log.info("All CSN's present and accounted for.")
#
# Print the convergence report
#
log.info('Measuring convergence...')
measureConvergence(topology)
#
# Test complete
#
log.info('No lingering entries.')
log.info('Pass %d complete.' % (loop + 1))
elapsed_tm = int(time.time() - TEST_START)
convtime = str(datetime.timedelta(seconds=elapsed_tm))
log.info('Entire test ran for: ' + convtime)
loop += 1
log.info('Test PASSED')
def test_MMR_Integrity(topology):
"""Apply load to 4 masters at the same time. Perform adds and deletes.
If any updates are missed we will see an error 32 in the access logs or
we will have entries left over once the test completes.
"""
loop = 0
ALL_REPLICAS = [topology.master1, topology.master2, topology.master3,
topology.master4,
topology.hub1, topology.hub2,
topology.consumer1, topology.consumer2,
topology.consumer3, topology.consumer4]
if TEST_CONVERGE_LATENCY:
try:
for inst in ALL_REPLICAS:
replica = inst.replicas.get(DEFAULT_SUFFIX)
replica.set('nsds5ReplicaReleaseTimeout', CONVERGENCE_TIMEOUT)
except ldap.LDAPError as e:
log.fatal('Failed to set replicas release timeout - error: %s' %
(str(e)))
assert False
if DEBUGGING:
# Enable Repl logging, and increase the max logs
try:
for inst in ALL_REPLICAS:
inst.enableReplLogging()
inst.modify_s("cn=config", [(ldap.MOD_REPLACE,
'nsslapd-errorlog-maxlogsperdir',
'5')])
except ldap.LDAPError as e:
log.fatal('Failed to set max logs - error: %s' % (str(e)))
assert False
while loop < MAX_LOOPS:
# Remove the current logs so we have a clean set of logs to check.
log.info('Pass %d...' % (loop + 1))
log.info("Removing logs...")
for inst in ALL_REPLICAS:
inst.deleteAllLogs()
# Fire off 4 threads to apply the load
log.info("Start adding/deleting: " + getDateTime())
startTime = time.time()
add_del_m1 = AddDelUsers(topology.master1)
add_del_m1.start()
add_del_m2 = AddDelUsers(topology.master2)
add_del_m2.start()
add_del_m3 = AddDelUsers(topology.master3)
add_del_m3.start()
add_del_m4 = AddDelUsers(topology.master4)
add_del_m4.start()
# Wait for threads to finish sending their updates
add_del_m1.join()
add_del_m2.join()
add_del_m3.join()
add_del_m4.join()
log.info("Finished adding/deleting entries: " + getDateTime())
#
# Loop checking for error 32's, and for convergence to complete
#
log.info("Waiting for replication to converge...")
while True:
# First check for error 32's
for inst in ALL_REPLICAS:
if inst.searchAccessLog(" err=32 "):
log.fatal('An add was missed on: ' + inst.serverid)
assert False
# Next check to see if the last update is in the access log
converged = True
for inst in ALL_REPLICAS:
if not inst.searchAccessLog(LAST_DN_M1) or \
not inst.searchAccessLog(LAST_DN_M2) or \
not inst.searchAccessLog(LAST_DN_M3) or \
not inst.searchAccessLog(LAST_DN_M4):
converged = False
break
if converged:
elapsed_tm = int(time.time() - startTime)
convtime = str(datetime.timedelta(seconds=elapsed_tm))
log.info('Replication converged at: ' + getDateTime() +
' - Elapsed Time: ' + convtime)
break
else:
# Check if replication is idle
replicas = [topology.master1, topology.master2,
topology.master3, topology.master4,
topology.hub1, topology.hub2]
if ReplTools.replIdle(replicas, DEFAULT_SUFFIX):
# Replication is idle - wait 30 secs for access log buffer
time.sleep(30)
# Now check the access log again...
converged = True
for inst in ALL_REPLICAS:
if not inst.searchAccessLog(LAST_DN_M1) or \
not inst.searchAccessLog(LAST_DN_M2) or \
not inst.searchAccessLog(LAST_DN_M3) or \
not inst.searchAccessLog(LAST_DN_M4):
converged = False
break
if converged:
elapsed_tm = int(time.time() - startTime)
convtime = str(datetime.timedelta(seconds=elapsed_tm))
log.info('Replication converged at: ' + getDateTime() +
' - Elapsed Time: ' + convtime)
break
else:
log.fatal('Stopping replication check: ' +
getDateTime())
log.fatal('Failure: Replication is complete, but we ' +
'never converged.')
assert False
# Sleep a bit before the next pass
time.sleep(3)
#
# Finally check the CSN's
#
log.info("Check the CSN's...")
if not ReplTools.checkCSNs(ALL_REPLICAS):
assert False
log.info("All CSN's present and accounted for.")
#
# Print the convergence report
#
log.info('Measuring convergence...')
measureConvergence(topology)
#
# Test complete
#
log.info('No lingering entries.')
log.info('Pass %d complete.' % (loop + 1))
elapsed_tm = int(time.time() - TEST_START)
convtime = str(datetime.timedelta(seconds=elapsed_tm))
log.info('Entire test ran for: ' + convtime)
loop += 1
log.info('Test PASSED')
