def setup(islandId):
sor = SOR()
hosts = sor.getHosts('purge-dispatcherV2')
if hosts is None or len(hosts) != 1:
print 'Did not find expected host for purge-dispatcherV2',hosts
return 1
purgeHost = hosts.pop()
mc = ManagementContainer.getInstance()
purgeDispatcher = Service('purge-dispatcherV2')
preCalcSvc = Service('purge-precalc-service')
try:
purgeDispatcher.invoke('stop',purgeHost)
preCalcSvc.invoke('stop',purgeHost)
os.system("cd /tmp; /bin/rm -rf /tmp/corpus /tmp/policystats-1; tar -xf endtoend-test-corpus.tar; mv /tmp/corpus /tmp/policystats-1");
os.system("sed -i 's/@enron.com/@policystats.com/g' /tmp/policystats-1/*");
setupCustomer(mc,islandId,corpus='/tmp/policystats-1',name='PolicyStatsTest',domain='policystats.com',recvDate='now')
return 0
finally:
purgeDispatcher.invoke('start',purgeHost)
preCalcSvc.invoke('start',purgeHost)
def enableIslandEDiscovery( islandId, enabled ):
mc = ManagementContainer.getInstance()
island = mc.getIslandManager().getIsland( islandId )
edMode = island.isEdiscoveryEnabled()
if edMode != enabled:
island.setEdiscoveryEnabled( enabled )
mc.getIslandManager().updateIsland(island)
res = os.system( '/opt/ems/bin/amp cluster -l | grep -v FeedOn | grep -E \" .*' + str( islandId ) + ' .*\" > /tmp/cluster.out')
if res != 0:
return ERROR_CLUSTER_FAILED
linedata = open( '/tmp/cluster.out', 'r' ).read()
items = linedata.split()
enderServiceName = items[5]
if island.getPlatformVersion().endswith( '-solr' ):
enderServiceName = 'solrproxy-' + enderServiceName
else:
enderServiceName = 'fastproxy-' + enderServiceName
proxyHost = items[10]
print 'restarting proxy \"' + enderServiceName + '\" on \"' + proxyHost + '\" to ensure that island capability cache is in synch'
proxy = Service( enderServiceName )
proxy.invoke( 'restart', proxyHost )
print 'proxy restarted'
return edMode
def prepDataForPurge(mc, custid, purgeHost):
pm = mc.getRetentionPolicyManager()
rp = pm.getLongestRetentionPolicy(custid)
rp.setDaysRetention(0)
pm.savePolicy(rp)
syncService = Service("purge-amdb-sync")
syncService.invoke("restart", purgeHost)
conn = None
done = False
maxTries = 24
while not done and maxTries > 0:
stmt = None
try:
conn = mc.getDBConnection(ManagementContainer.AM_POOL_NAME)
stmt = conn.createStatement()
rs = stmt.executeQuery("select retain_days from retention_policies where policy_id = %d" % rp.getId())
done = rs is not None and rs.next() and rs.getInt(1) == 0
finally:
if conn is not None:
conn.rollback()
if rs is not None:
rs.close()
if stmt is not None:
stmt.close()
if conn is not None:
mc.returnDBConnection(conn, ManagementContainer.AM_POOL_NAME)
if not done:
print "Waiting for retention policy to update on AM DB"
time.sleep(10)
maxTries = maxTries - 1
if maxTries <= 0:
raise Exception("purge-amdb-sync did not update policies")
def setup(islandId):
sor = SOR()
hosts = sor.getHosts("purge-dispatcherV2")
if hosts is None or len(hosts) != 1:
print "Did not find expected host for purge-dispatcherV2", hosts
return 1
purgeHost = hosts.pop()
mc = ManagementContainer.getInstance()
purgeDispatcher = Service("purge-dispatcherV2")
preCalcSvc = Service("purge-precalc-service")
try:
purgeDispatcher.invoke("stop", purgeHost)
preCalcSvc.invoke("stop", purgeHost)
os.system(
"cd /tmp; /bin/rm -rf /tmp/corpus /tmp/purge-1; tar -xf endtoend-test-corpus.tar; mv /tmp/corpus /tmp/purge-1"
)
os.system("sed -i 's/@enron.com/@purge1.com/g' /tmp/purge-1/*")
setupCustomer(mc, islandId, corpus="/tmp/purge-1", name="purge1", domain="purge1.com", recvDate="now")
return 0
finally:
purgeDispatcher.invoke("start", purgeHost)
preCalcSvc.invoke("start", purgeHost)
def enableIslandEDiscovery( islandId, enabled ):
file = File('/tmp/cluster_tmp.out')
ampout = PrintStream(FileOutputStream(file))
amptool = ActiveMailboxPartitionTool()
amptool.setOutputStream(ampout)
mc = ManagementContainer.getInstance()
island = mc.getIslandManager().getIsland( islandId )
edMode = island.isEdiscoveryEnabled()
if edMode != enabled:
island.setEdiscoveryEnabled( enabled )
mc.getIslandManager().updateIsland(island)
result = amptool.runCommand('cluster',array(['-l'],String))
cmdline = 'cat /tmp/cluster_tmp.out | grep -v FeedOn | grep -E \" .*' + str( islandId ) + ' .*\" > /tmp/cluster.out'
os.system(cmdline)
if result is False:
return ERROR_CLUSTER_FAILED
linedata = open( '/tmp/cluster.out', 'r' ).read()
items = linedata.split()
enderServiceName = items[5]
if island.getPlatformVersion().endswith( '-solr' ):
enderServiceName = 'solrproxy-' + enderServiceName
else:
enderServiceName = 'fastproxy-' + enderServiceName
proxyHost = items[10]
print 'restarting proxy \"' + enderServiceName + '\" on \"' + proxyHost + '\" to ensure that island capability cache is in synch'
proxy = Service( enderServiceName )
proxy.invoke( 'restart', proxyHost )
print 'proxy restarted'
return edMode
def ensureServiceUp(serviceName, hostList):
svc = Service(serviceName)
result = svc.invoke('start',hostList)
for r in result:
# print result[r]
# print result[r].isRunning()
if result[r].isRunning() != 1:
msg = 'service '+ serviceName+ ' failed to start on host '+ r
print msg
raise Exception(msg)
def enderStop(hostList, serviceName):
print 'stopping service', serviceName,'on', hostList
svc = Service(serviceName)
results = svc.invoke('stop',hostList)
for r in results :
result = results[r]
if result.isRunning() != 0:
msg = 'service '+ serviceName+ ' failed to stop on host '+ r
print msg
raise Exception(msg)
def upServices(serviceName, hostList):
upServers = []
svc = Service(serviceName)
result = svc.invoke('status',hostList)
for r in result:
# print result[r]
# print result[r].getRetCode()
# print result[r].isRunning()
if result[r].isRunning() == 1 :
upServers.append(r)
return upServers
def enderStart(hostList, serviceName):
print 'starting service', serviceName,'on', hostList
svc = Service(serviceName)
results = svc.invoke('start',hostList)
for r in results :
result = results[r]
print 'running', result.isRunning()
if result.isRunning() != 1 :
msg = 'service '+ serviceName+ ' failed to start on host '+ hostList
print msg
raise Exception(msg)
def test(islandId, numMessages, esHosts):
mc = ManagementContainer.getInstance()
lastVal = None
custId = None
service = Service("storageimporter")
msgs = None
try:
lastVal, nextVal = setMessageId(mc, Integer.MAX_VALUE - 2)
print "lastVal,nextVal", lastVal, nextVal
print "restart storageimporters", esHosts
service.invoke("restart", esHosts)
custId = setupCustomer(mc, islandId)
msgs = findMessages(mc, custId, numMessages)
print "created messages", msgs
checkSearchStatus(mc, msgs, custId)
ism = mc.getIndexSearchManager()
isc = IndexSearchConstraint(None, None)
isc.sortBy("storageid", True)
isc.constrainByMinimumStorageId(0)
isc.constrainByIsland(mc.getIslandManager().getIsland(islandId))
print "DBG min", 0
query = "customerid:" + str(custId)
sr = ism.searchAndWrap(query, isc, CallerApp.INTERNAL)
if sr.getDocCount() != 10:
print >> sys.stderr, "Failed to find 10 documents when minStorageId = 0, found", sr.getDocCount()
return 1
print "DBG min", Integer.MAX_VALUE - 1
isc.constrainByMinimumStorageId(Integer.MAX_VALUE - 1)
sr = ism.searchAndWrap(query, isc, CallerApp.INTERNAL)
if sr.getDocCount() != 9:
print >> sys.stderr, "Failed to find 9 documents when minStorageId = ", Integer.MAX_VALUE - 1, ",found", sr.getDocCount()
return 1
print "DBG min", Integer.MAX_VALUE + 1
isc.constrainByMinimumStorageId(Integer.MAX_VALUE + 1)
sr = ism.searchAndWrap(query, isc, CallerApp.INTERNAL)
if sr.getDocCount() != 7:
print >> sys.stderr, "Failed to find 7 documents when minStorageId = ", Integer.MAX_VALUE + 1, ",found", sr.getDocCount()
return 1
finally:
if msgs is not None:
for msg in msgs:
purgeMessage(mc, msg, custId)
if custId is not None:
mc.getCustomerManager().deleteCustomers([custId])
if lastVal is not None:
lastVal, nextVal = setMessageId(mc, lastVal)
print "lastVal,nextVal", lastVal, nextVal
print "restart storageimporters", esHosts
service.invoke("restart", esHosts)
return 0
def runit(action, svc_name, system_name):
svc = Service(svc_name)
r = svc.invoke(action, system_name)
status = "error (pid 0)"
if r.isFailed() :
status = "error (pid 0)"
elif r.isRunning() :
status = "run (pid x)"
else :
status = "down (pid 0)"
print system_name, svc_name, status
def upgradeMaster(islandID):
clusterMgr = mc.getClusterManager()
clusters = clusterMgr.enumClusters(islandID)
for cluster in clusters:
locs = clusterMgr.enumClusterLocations(cluster.getID())
hosts = []
for loc in locs:
masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP))
masterHost = masterURL.getHost()
hosts.append(masterHost)
service = Service("ems-solr")
service.setThreadCount(1)
service.invoke("stop", hosts)
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "save-index"], hosts)
if not failures.isEmpty():
print "WARN: Failed to backup the index on all hosts. Check logs.", failures
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "upgrade-index"], hosts)
if not failures.isEmpty():
print "Failed to start the backup job running on all hosts. Check logs.", failures
return 1
failures = runJob(
["/usr/local/bin/ender", "remote-action", "upgrade-support", "upgrade-status"],
hosts,
sleepTime=120,
maxWaits=43200,
interimResult=3,
)
if not failures.isEmpty():
print "ERROR: upgrade job failed. Upgrade aborted", failures
return 1
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "remove-solr-service"], hosts)
if not failures.isEmpty():
print "ERROR: unable to remove solr service from all hosts", failures
return 1
print "Please check for any errors in the output before proceeding"
print "If it appears the Solr index has been upgraded successfully. Please proceed to install the new version of Solr and then run the validate-master step"
return 0
def setup(islandId,numMessages):
mc = ManagementContainer.getInstance()
svc = Service('storageimporter')
# remote test
setPartitionsWriteable(mc,True,True) # local readonly
setPartitionsWriteable(mc,False,False) # remote writeable
svc.invoke('restart','localhost')
try:
os.system("cd /tmp; /bin/rm -rf /tmp/corpus /tmp/rgm-1; tar -xf endtoend-test-corpus.tar; mv /tmp/corpus /tmp/rgm-1");
os.system("sed -i 's/@enron.com/@rgm1.com/g' /tmp/rgm-1/*");
custId = setupCustomer(mc,islandId,corpus='/tmp/rgm-1',name='rgm-remote',domain='rgm1.com',recvDate='now')
msgs = findMessages(mc,custId,numMessages)
partId = msgs.iterator().next().getPartitionId()
partition = mc.getPartitionManager().getPartition(partId)
if partition.isLocal():
print 'Expected messages to be stored remotely but found message on partition',partId
return 1
finally:
setPartitionsWriteable(mc,False,True)
setPartitionsWriteable(mc,False,False)
# local test
setPartitionsWriteable(mc,False,True) # local writeable
setPartitionsWriteable(mc,True,False) # remote readonly
svc.invoke('restart','localhost')
try:
os.system("cd /tmp; /bin/rm -rf /tmp/corpus /tmp/rgm-2; tar -xf endtoend-test-corpus.tar; mv /tmp/corpus /tmp/rgm-2");
os.system("sed -i 's/@enron.com/@rgm2.com/g' /tmp/rgm-2/*");
custId = setupCustomer(mc,islandId,corpus='/tmp/rgm-2',name='rgm-local',domain='rgm2.com',recvDate='now')
msgs = findMessages(mc,custId,numMessages)
partId = msgs.iterator().next().getPartitionId()
partition = mc.getPartitionManager().getPartition(partId)
if not partition.isLocal():
print 'Expected messages to be stored locally but found message on partition',partId
return 1
finally:
setPartitionsWriteable(mc,False,True)
setPartitionsWriteable(mc,False,False)
return 0
def stopPurge(purgeHost, cleanup = True):
purgeDispatcher = Service('purge-dispatcherV2')
preCalcSvc = Service('purge-precalc-service')
purgeDispatcher.invoke('stop',purgeHost)
preCalcSvc.invoke('stop',purgeHost)
if cleanup:
for f in os.listdir('/ems/shared/purge'):
parent = os.path.join('/ems/shared/purge',f)
print 'checking path:', parent
sentinel = os.path.join(parent,'sentinel')
if os.path.exists(sentinel):
print 'removing:', sentinel
os.remove(sentinel)
def startCache(cacheHost):
cacheHandler = Service('voltdb')
cacheHandler.invoke('start',cacheHost)
print time.asctime(),'Cache on host '+ cacheHost + ' has started'
def fullStatus(islandID):
islandMgr = mc.getIslandManager()
island = islandMgr.getIsland(int(islandID))
print "INFO: island status for island", islandID
print "INFO: search URL", islandID, island.getSearchURL()
print "INFO: search Parmeters", islandID, island.getSearchParms()
print "INFO: cluster and shard status for island", islandID
clusterMgr = mc.getClusterManager()
clusters = clusterMgr.enumClusters(islandID)
masters = []
slaves = []
masterURLs = []
slaveURLs = []
for cluster in clusters:
print "INFO: cluster", cluster.getID(), "F:", cluster.isFeedEnabled(), "U:", cluster.isUpdateEnabled(), "P:", cluster.isPurgeEnabled()
locs = clusterMgr.enumClusterLocations(cluster.getID())
for loc in locs:
print "INFO: location", loc.getId(), "F:", loc.isFeedEnabled(), "U:", loc.isUpdateEnabled(), "P:", loc.isPurgeEnabled(), "M:", loc.isManagementEnabled()
slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP))
slaveHost = slaveURL.getHost()
slaves.append(slaveHost)
masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP))
masterHost = masterURL.getHost()
masters.append(masterHost)
masterURL = masterURL.toString() + "/admin/system"
masterURLs.append(masterURL)
slaveURL = slaveURL.toString() + "/admin/system"
slaveURLs.append(slaveURL)
print "INFO: Service Status for island", islandID
service = Service("ems-solr")
service.setThreadCount(1)
responses = service.invoke("status", masters)
for key in responses.keySet():
sr = responses.get(key)
print "INFO: Master", key, sr.getStdout()
responses = service.invoke("status", slaves)
for key in responses.keySet():
sr = responses.get(key)
print "INFO: Slave", key, sr.getStdout()
print "INFO: Version Info for island", islandID
for masterURL in masterURLs:
masterF = None
try:
masterF = urllib.urlopen(masterURL)
except:
print "WARN: Solr not running on", masterURL
continue
try:
masterData = masterF.read()
look = '<str name="solr-spec-version">'
idx = masterData.find(look)
idx = idx + len(look)
end = masterData.find("</str>", idx)
print "INFO:", masterURL, masterData[idx:end]
finally:
masterF.close()
for slaveURL in slaveURLs:
slaveF = None
try:
slaveF = urllib.urlopen(slaveURL)
except:
print "WARN: Solr not running on", slaveURL
continue
try:
slaveData = slaveF.read()
look = '<str name="solr-spec-version">'
idx = slaveData.find(look)
idx = idx + len(look)
end = slaveData.find("</str>", idx)
print "INFO:", slaveURL, slaveData[idx:end]
finally:
slaveF.close()
print "INFO: Upgrade activities"
ulog = "/tmp/.solr_upgrade_log_" + str(islandID)
if os.path.exists(ulog):
for line in open(ulog):
print "INFO:", line.strip()
else:
print "INFO: No activity log found:", ulog
def upgradeSlave(islandID):
clusterMgr = mc.getClusterManager()
clusters = clusterMgr.enumClusters(islandID)
hosts = []
for cluster in clusters:
locs = clusterMgr.enumClusterLocations(cluster.getID())
for loc in locs:
slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP))
slaveHost = slaveURL.getHost()
masterURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_MASTER_HOST_URL_PROP))
masterURL = masterURL.toString() + "/m1Monitor?command=register"
slaveURL = slaveURL.toString()
slaveURL = slaveURL + "/m1Monitor?command=shutdown"
try:
slaveF = urllib.urlopen(slaveURL)
except:
print "WARN: slave down, expected up", slaveURL
logOutput(slaveF)
slaveF.close()
try:
masterF = urllib.urlopen(masterURL)
except:
print "ERROR: master down, must be up", masterURL
return 1
logOutput(masterF)
masterF.close()
hosts.append(slaveHost)
islandMgr = mc.getIslandManager()
retries = 5
done = False
while not done and retries > 0:
done = True
island = islandMgr.getIsland(islandID)
parms = island.getSearchParms()
for cluster in clusters:
locs = clusterMgr.enumClusterLocations(cluster.getID())
for loc in locs:
slaveURL = URL(loc.getClusterLocationProperty(SolrClusterAdapter.SOLR_SLAVE_HOST_URL_PROP)).toString()
if slaveURL in parms:
print slaveURL, "still in search params, waiting for update"
done = False
retries = retries - 1
if not done and retries == 0:
print "Unable to start slave upgrade because search URL does not show masters handling all search requests"
return 1
print "Stopping Solr on slaves"
service = Service("ems-solr")
service.setThreadCount(1)
service.invoke("stop", hosts)
print "Removing Solr service from slaves to prepare for upgrade"
failures = runJob(["/usr/local/bin/ender", "remote-action", "upgrade-support", "remove-solr-service"], hosts)
if failures.isEmpty():
print "INFO: Search is now successfully running on masters, please use Extropy to upgrade slaves"
else:
print "WARN: check logs for errors before upgrading slaves. Host(s) reporting falilure:", failures
return 0
def test(sourceIslandId,targetIslandId,numMessages):
rc = 1
mc = ManagementContainer.getInstance()
cm = mc.getCustomerManager()
srcIsland = mc.getIslandManager().getIsland(sourceIslandId)
tgtIsland = mc.getIslandManager().getIsland(targetIslandId)
tgtEDMode = None
custid = None
proxy = Service('solrproxy-Island102Cluster1')
try:
tgtEDMode = tgtIsland.isEdiscoveryEnabled()
# turn on e-discovery on target island
tgtIsland.setEdiscoveryEnabled(True)
mc.getIslandManager().updateIsland(tgtIsland)
print 'restarting proxy to ensure that island capability cache is in synch'
proxy.invoke('restart','work-3')
print 'proxy restarted'
# setup customer
custid = setupCustomer(mc,sourceIslandId)
caps = mc.getCustomerManager().getCustomerCapabilities(int(custid))
print custid,'ediscovery is',caps.getBooleanCapability(Capabilities.CAP_ALLOW_ARCHIVE_EDISCOVERY)
rc = testMigrate(custid,sourceIslandId,targetIslandId,numMessages)
if rc != 0:
return rc
# delete customer
cm.deleteCustomers([custid])
custid = None
# stop the storage importer
storageImporter = Service('storageimporter')
storageImporter.invoke('stop','localhost')
# turn off e-discovery on target island
tgtEDMode = tgtIsland.isEdiscoveryEnabled()
tgtIsland.setEdiscoveryEnabled(False)
mc.getIslandManager().updateIsland(tgtIsland)
print 'restarting proxy to ensure that island capability cache is in synch'
proxy.invoke('restart','work-3')
print 'proxy restarted'
# create a customer, do not yet import mail
custid = setupCustomer(mc,sourceIslandId)
# turn off ediscovery for customer
caps = mc.getCustomerManager().getCustomerCapabilities(int(custid))
caps.setBooleanCapability(Capabilities.CAP_ALLOW_ARCHIVE_EDISCOVERY,False)
mc.getCustomerManager().saveCustomerCapabilities(caps)
caps = mc.getCustomerManager().getCustomerCapabilities(int(custid))
print custid,'ediscovery is',caps.getBooleanCapability(Capabilities.CAP_ALLOW_ARCHIVE_EDISCOVERY)
# import the messages after change to customer caps
storageImporter.invoke('start','localhost')
rc = testMigrate(custid,sourceIslandId,targetIslandId,numMessages)
finally:
if custid is not None:
cm.deleteCustomers([custid])
if tgtIsland is not None and tgtIsland.isEdiscoveryEnabled() != tgtEDMode:
tgtIsland.setEdiscoveryEnabled(tgtEDMode)
mc.getIslandManager().updateIsland(tgtIsland)
print 'restarting proxy to ensure that island capability cache is in synch'
proxy.invoke('restart','work-3')
print 'proxy restarted'
return rc
def startSearchDuplicateCleanup(searchDuplicateCleanupHost):
searchDuplicateCleanup = Service('search-duplicate-cleanup')
searchDuplicateCleanup.invoke('start',searchDuplicateCleanupHost)
def test(islandId, numMessages, custName):
sor = SOR()
hosts = sor.getHosts("purge-dispatcherV2")
if hosts is None or len(hosts) != 1:
print "Did not find expected host for purge-dispatcherV2", hosts
sys.exit(1)
purgeHost = hosts.pop()
rc = True
mc = ManagementContainer.getInstance()
cm = mc.getCustomerManager()
amsm = mc.getActiveMailboxStoreManager()
purgeDispatcher = Service("purge-dispatcherV2")
preCalcSvc = Service("purge-precalc-service")
purgeDispatcher.invoke("stop", purgeHost)
preCalcSvc.invoke("stop", purgeHost)
custid = findCustomer(custName)
if custid < 0:
print "test failed because customer", custName, "was not found"
return False
prepDataForPurge(mc, custid, purgeHost)
preCalcSvc.invoke("start", purgeHost)
msgs = findMessages(mc, custid, numMessages)
msg = selectMessage(mc, msgs)
purgeMessage(mc, msg, custid)
partition = mc.getPartitionManager().getPartition(msg.getPartitionId())
tm = amsm.findMessages(
[
SearchConstraint(
IActiveMailboxStoreManager.PROP_CUST_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, int(custid)
),
SearchConstraint(
IActiveMailboxStoreManager.PROP_MSG_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, msg.getMessageId()
),
],
partition,
)
if not tm.isEmpty():
print "purge failed to delete message", msg
sys.exit(1)
for p in mc.getPartitionManager().listPartitions():
amsm.purgeRequest(p)
now = Date()
purgeDispatcher.invoke("start", purgeHost)
done = checkLastPurge(mc, now)
if done:
for msg in msgs:
partition = mc.getPartitionManager().getPartition(msg.getPartitionId())
tm = amsm.findMessages(
[
SearchConstraint(
IActiveMailboxStoreManager.PROP_CUST_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, int(custid)
),
SearchConstraint(
IActiveMailboxStoreManager.PROP_MSG_ID,
SearchConstraintOperator.CONSTRAINT_EQUALS,
msg.getMessageId(),
),
],
partition,
)
if not tm.isEmpty():
print "purge failed to delete", msg
rc = False
else:
rc = False
if rc is True:
print "purge successful"
else:
print "purge failed"
sys.exit(rc is True)
def startPurge(purgeHost):
purgeDispatcher = Service('purge-dispatcherV2')
preCalcSvc = Service('purge-precalc-service')
purgeDispatcher.invoke('start',purgeHost)
preCalcSvc.invoke('start',purgeHost)
def test(islandId):
global purgeClient
global preCalcSvc
global purgeDispatcher
global purgeHost
sor = SOR()
hosts = sor.getHosts('purge-dispatcherV2')
if hosts is None or len(hosts) != 1:
print 'Did not find expected host for purge-dispatcherV2',hosts
sys.exit(1)
purgeHost = hosts.pop()
# ran into some shell issues. this apparently fixes it
os.system("cd")
rc = False
cm = mc.getCustomerManager()
purgeDispatcher = Service('purge-dispatcherV2')
preCalcSvc = Service('purge-precalc-service')
purgeClient = Service('purge-client')
debug("test: killing precalc and purge-dispatcher services on " + purgeHost + " and purge on local machine")
purgeDispatcher.invoke('stop',purgeHost)
preCalcSvc.invoke('stop',purgeHost)
purgeClient.invoke('stop',localhost)
if not os.path.exists(sentinelSpoolDir): os.makedirs(sentinelSpoolDir)
try:
debug("test: creating new customer")
# order of test cases (all have valid sentinel file in shared - do not change order!):
# valid batch in shared directory, completely corrupt copy in bigdisk
# valid batch in shared directory, no copy in bigdisk
# completely corrupted batch in shared directory, completely corrupt copy in bigdisk
# completely corrupted batch in shared directory, no copy in bigdisk
# no batch in shared directory, no copy in bigdisk
# valid batch in shared, no reading permissions copy in bigdisk
# partially valid batch in shared, partially valid copy in bigdisk
# partially valid batch in shared, completely corrupt copy in bigdisk
# partially valid batch in shared, no copy in bigdisk
# valid batch in shared directory, partially valid copy in bigdisk
#
# regression should be done in last test case!
rc = genericTest()
if rc: rc = genericTest(shouldDeleteSpoolBatch=True)
if rc: rc = genericTest(shouldMuckUpWorkBatch=True)
if rc: rc = genericTest(shouldMuckUpWorkBatch=True, shouldDeleteSpoolBatch=True)
if rc: rc = genericTest(shouldDeleteWorkBatch=True, shouldDeleteSpoolBatch=True, shouldMuckUpBatchFileInSpoolDir=False)
if rc: rc = genericTest(shouldChangePermissions=True)
if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldPartialSpoolBatch=True)
if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldMuckUpWorkBatch=True)
if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldDeleteSpoolBatch=True)
if rc: rc = genericTest(shouldPartialSpoolBatch=True)# , shouldDoRegression=True)
finally:
# clean up
cleanPurgeDirs()
debug("test: restarting purge dispatcher, purge, and precalc services")
purgeDispatcher.invoke('start',purgeHost)
preCalcSvc.invoke('start',purgeHost)
purgeClient.invoke('start',localhost)
debug("test: returning " + str(rc is not True))
log.close()
sys.exit(rc is not True)
def test(islandId,numMessages,custName):
sor = SOR()
hosts = sor.getHosts('purge-dispatcherV2')
if hosts is None or len(hosts) != 1:
print 'Did not find expected host for purge-dispatcherV2',hosts
sys.exit(1)
purgeHost = hosts.pop()
mc = ManagementContainer.getInstance()
cm = mc.getCustomerManager()
amsm = mc.getActiveMailboxStoreManager()
preCalcSvc = Service('purge-precalc-service')
preCalcSvc.invoke('stop',purgeHost)
custid = findCustomer(custName)
if custid < 0:
print 'test failed because customer',custName,'was not found'
return False
clearPolicyStats(custid)
rpm = mc.getRetentionPolicyManager()
rp = rpm.getLongestRetentionPolicy(custid)
pmm = mc.getPolicyMetaManager()
policyStats = pmm.getPolicyStats(custid)
if not policyStats.isEmpty():
print 'test failed because policy stats was not empty for customer',custid
return False
msgs = findMessages(mc,custid,numMessages)
for p in mc.getPartitionManager().listPartitions():
amsm.purgeRequest(p)
preCalcSvc.invoke('start',purgeHost)
# wait for the stats to get generated
maxTries = 60
while not areStatsGenerated(custid) and maxTries > 0:
print 'Waiting for pre-calc service to generate stats...'
time.sleep(10)
maxTries = maxTries - 1
if not areStatsGenerated(custid):
print 'test failed because policy stats were not generated within time alotted'
return False
os.system('ssh -o StrictHostKeyChecking=no %s "bash /opt/ems/bin/policy_stats"' % purgeHost)
policyStats = pmm.getPolicyStats(custid)
if policyStats.isEmpty():
print 'test failed because policy stats was empty for customer',custid
return False
if not policyStats.containsKey(rp.getId()):
print 'test failed becasue policy stats did not contain an entry for id',rp.getId()
return False
stat = policyStats.get(rp.getId())
print 'users',stat.getNumUsers(),'size',stat.getMessageSize(),'count',stat.getMessageCount()
if numMessages != stat.getMessageCount():
print 'test failed because expected',numMessages,'but only found',stat.getMessageCount(),'in stats'
return True
def stopCache(cacheHost):
cacheHandler = Service('voltdb')
cacheHandler.invoke('stop',cacheHost)
time.sleep(10)
print time.asctime(),'Cache on host '+ cacheHost + ' has stopped'
