def setup(islandId):
sor = SOR()
hosts = sor.getHosts("purge-dispatcherV2")
if hosts is None or len(hosts) != 1:
print "Did not find expected host for purge-dispatcherV2", hosts
return 1
purgeHost = hosts.pop()
mc = ManagementContainer.getInstance()
purgeDispatcher = Service("purge-dispatcherV2")
preCalcSvc = Service("purge-precalc-service")
try:
purgeDispatcher.invoke("stop", purgeHost)
preCalcSvc.invoke("stop", purgeHost)
os.system(
"cd /tmp; /bin/rm -rf /tmp/corpus /tmp/purge-1; tar -xf endtoend-test-corpus.tar; mv /tmp/corpus /tmp/purge-1"
)
os.system("sed -i 's/@enron.com/@purge1.com/g' /tmp/purge-1/*")
setupCustomer(mc, islandId, corpus="/tmp/purge-1", name="purge1", domain="purge1.com", recvDate="now")
return 0
finally:
purgeDispatcher.invoke("start", purgeHost)
preCalcSvc.invoke("start", purgeHost)
def setup(islandId):
sor = SOR()
hosts = sor.getHosts('purge-dispatcherV2')
if hosts is None or len(hosts) != 1:
print 'Did not find expected host for purge-dispatcherV2',hosts
return 1
purgeHost = hosts.pop()
mc = ManagementContainer.getInstance()
purgeDispatcher = Service('purge-dispatcherV2')
preCalcSvc = Service('purge-precalc-service')
try:
purgeDispatcher.invoke('stop',purgeHost)
preCalcSvc.invoke('stop',purgeHost)
os.system("cd /tmp; /bin/rm -rf /tmp/corpus /tmp/policystats-1; tar -xf endtoend-test-corpus.tar; mv /tmp/corpus /tmp/policystats-1");
os.system("sed -i 's/@enron.com/@policystats.com/g' /tmp/policystats-1/*");
setupCustomer(mc,islandId,corpus='/tmp/policystats-1',name='PolicyStatsTest',domain='policystats.com',recvDate='now')
return 0
finally:
purgeDispatcher.invoke('start',purgeHost)
preCalcSvc.invoke('start',purgeHost)
def test(islandId, numMessages, custName):
sor = SOR()
hosts = sor.getHosts("purge-dispatcherV2")
if hosts is None or len(hosts) != 1:
print "Did not find expected host for purge-dispatcherV2", hosts
sys.exit(1)
purgeHost = hosts.pop()
rc = True
mc = ManagementContainer.getInstance()
cm = mc.getCustomerManager()
amsm = mc.getActiveMailboxStoreManager()
purgeDispatcher = Service("purge-dispatcherV2")
preCalcSvc = Service("purge-precalc-service")
purgeDispatcher.invoke("stop", purgeHost)
preCalcSvc.invoke("stop", purgeHost)
custid = findCustomer(custName)
if custid < 0:
print "test failed because customer", custName, "was not found"
return False
prepDataForPurge(mc, custid, purgeHost)
preCalcSvc.invoke("start", purgeHost)
msgs = findMessages(mc, custid, numMessages)
msg = selectMessage(mc, msgs)
purgeMessage(mc, msg, custid)
partition = mc.getPartitionManager().getPartition(msg.getPartitionId())
tm = amsm.findMessages(
[
SearchConstraint(
IActiveMailboxStoreManager.PROP_CUST_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, int(custid)
),
SearchConstraint(
IActiveMailboxStoreManager.PROP_MSG_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, msg.getMessageId()
),
],
partition,
)
if not tm.isEmpty():
print "purge failed to delete message", msg
sys.exit(1)
for p in mc.getPartitionManager().listPartitions():
amsm.purgeRequest(p)
now = Date()
purgeDispatcher.invoke("start", purgeHost)
done = checkLastPurge(mc, now)
if done:
for msg in msgs:
partition = mc.getPartitionManager().getPartition(msg.getPartitionId())
tm = amsm.findMessages(
[
SearchConstraint(
IActiveMailboxStoreManager.PROP_CUST_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, int(custid)
),
SearchConstraint(
IActiveMailboxStoreManager.PROP_MSG_ID,
SearchConstraintOperator.CONSTRAINT_EQUALS,
msg.getMessageId(),
),
],
partition,
)
if not tm.isEmpty():
print "purge failed to delete", msg
rc = False
else:
rc = False
if rc is True:
print "purge successful"
else:
print "purge failed"
sys.exit(rc is True)
def test(islandId):
global purgeClient
global preCalcSvc
global purgeDispatcher
global purgeHost
sor = SOR()
hosts = sor.getHosts('purge-dispatcherV2')
if hosts is None or len(hosts) != 1:
print 'Did not find expected host for purge-dispatcherV2',hosts
sys.exit(1)
purgeHost = hosts.pop()
# ran into some shell issues. this apparently fixes it
os.system("cd")
rc = False
cm = mc.getCustomerManager()
purgeDispatcher = Service('purge-dispatcherV2')
preCalcSvc = Service('purge-precalc-service')
purgeClient = Service('purge-client')
debug("test: killing precalc and purge-dispatcher services on " + purgeHost + " and purge on local machine")
purgeDispatcher.invoke('stop',purgeHost)
preCalcSvc.invoke('stop',purgeHost)
purgeClient.invoke('stop',localhost)
if not os.path.exists(sentinelSpoolDir): os.makedirs(sentinelSpoolDir)
try:
debug("test: creating new customer")
# order of test cases (all have valid sentinel file in shared - do not change order!):
# valid batch in shared directory, completely corrupt copy in bigdisk
# valid batch in shared directory, no copy in bigdisk
# completely corrupted batch in shared directory, completely corrupt copy in bigdisk
# completely corrupted batch in shared directory, no copy in bigdisk
# no batch in shared directory, no copy in bigdisk
# valid batch in shared, no reading permissions copy in bigdisk
# partially valid batch in shared, partially valid copy in bigdisk
# partially valid batch in shared, completely corrupt copy in bigdisk
# partially valid batch in shared, no copy in bigdisk
# valid batch in shared directory, partially valid copy in bigdisk
#
# regression should be done in last test case!
rc = genericTest()
if rc: rc = genericTest(shouldDeleteSpoolBatch=True)
if rc: rc = genericTest(shouldMuckUpWorkBatch=True)
if rc: rc = genericTest(shouldMuckUpWorkBatch=True, shouldDeleteSpoolBatch=True)
if rc: rc = genericTest(shouldDeleteWorkBatch=True, shouldDeleteSpoolBatch=True, shouldMuckUpBatchFileInSpoolDir=False)
if rc: rc = genericTest(shouldChangePermissions=True)
if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldPartialSpoolBatch=True)
if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldMuckUpWorkBatch=True)
if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldDeleteSpoolBatch=True)
if rc: rc = genericTest(shouldPartialSpoolBatch=True)# , shouldDoRegression=True)
finally:
# clean up
cleanPurgeDirs()
debug("test: restarting purge dispatcher, purge, and precalc services")
purgeDispatcher.invoke('start',purgeHost)
preCalcSvc.invoke('start',purgeHost)
purgeClient.invoke('start',localhost)
debug("test: returning " + str(rc is not True))
log.close()
sys.exit(rc is not True)
def test(islandId,numMessages,custName):
sor = SOR()
hosts = sor.getHosts('purge-dispatcherV2')
if hosts is None or len(hosts) != 1:
print 'Did not find expected host for purge-dispatcherV2',hosts
sys.exit(1)
purgeHost = hosts.pop()
mc = ManagementContainer.getInstance()
cm = mc.getCustomerManager()
amsm = mc.getActiveMailboxStoreManager()
preCalcSvc = Service('purge-precalc-service')
preCalcSvc.invoke('stop',purgeHost)
custid = findCustomer(custName)
if custid < 0:
print 'test failed because customer',custName,'was not found'
return False
clearPolicyStats(custid)
rpm = mc.getRetentionPolicyManager()
rp = rpm.getLongestRetentionPolicy(custid)
pmm = mc.getPolicyMetaManager()
policyStats = pmm.getPolicyStats(custid)
if not policyStats.isEmpty():
print 'test failed because policy stats was not empty for customer',custid
return False
msgs = findMessages(mc,custid,numMessages)
for p in mc.getPartitionManager().listPartitions():
amsm.purgeRequest(p)
preCalcSvc.invoke('start',purgeHost)
# wait for the stats to get generated
maxTries = 60
while not areStatsGenerated(custid) and maxTries > 0:
print 'Waiting for pre-calc service to generate stats...'
time.sleep(10)
maxTries = maxTries - 1
if not areStatsGenerated(custid):
print 'test failed because policy stats were not generated within time alotted'
return False
os.system('ssh -o StrictHostKeyChecking=no %s "bash /opt/ems/bin/policy_stats"' % purgeHost)
policyStats = pmm.getPolicyStats(custid)
if policyStats.isEmpty():
print 'test failed because policy stats was empty for customer',custid
return False
if not policyStats.containsKey(rp.getId()):
print 'test failed becasue policy stats did not contain an entry for id',rp.getId()
return False
stat = policyStats.get(rp.getId())
print 'users',stat.getNumUsers(),'size',stat.getMessageSize(),'count',stat.getMessageCount()
if numMessages != stat.getMessageCount():
print 'test failed because expected',numMessages,'but only found',stat.getMessageCount(),'in stats'
return True
while maxAttempts > 0:
os.system('ssh -o StrictHostKeyChecking=no %s "sh /opt/voltdb/restart-cache"' % cacheHost)
if checkCacheReady(cacheHost,sleepTime=5): return
maxAttempts = maxAttempts - 1
print time.asctime(),'Failed to restart cache to working state'
raise Exception('Cache failed to restart')
#
# main test body
#
if __name__ == '__main__':
sor = SOR()
print time.asctime(),'looking for all boxes:', sor.getAllManifestHosts()
print time.asctime(),'looking for cache boxes:', sor.findHostByName('cache-')
cacheHosts = sor.getHosts('voltdb')
if cacheHosts is None or len(cacheHosts)!=3:
print time.asctime(),'Did not find expected host for voltdb',cacheHosts
sys.exit(1)
cacheHost = cacheHosts[2]
dbHosts = sor.getHosts('postgresql')
if dbHosts is None or len(dbHosts) != 1:
print time.asctime(),'Did not find expected host for postgres',dhHosts
dbHost = dbHosts[0]
print time.asctime(),'Database host is ',dbHost
rc_1 = rc_2 = 1
try:
restartCache(cacheHost)
rc_2 = test_2(cacheHost)