def setup(islandId):
sor = SOR()
hosts = sor.getHosts('purge-dispatcherV2')
if hosts is None or len(hosts) != 1:
print 'Did not find expected host for purge-dispatcherV2',hosts
return 1
purgeHost = hosts.pop()
mc = ManagementContainer.getInstance()
purgeDispatcher = Service('purge-dispatcherV2')
preCalcSvc = Service('purge-precalc-service')
try:
purgeDispatcher.invoke('stop',purgeHost)
preCalcSvc.invoke('stop',purgeHost)
os.system("cd /tmp; /bin/rm -rf /tmp/corpus /tmp/policystats-1; tar -xf endtoend-test-corpus.tar; mv /tmp/corpus /tmp/policystats-1");
os.system("sed -i 's/@enron.com/@policystats.com/g' /tmp/policystats-1/*");
setupCustomer(mc,islandId,corpus='/tmp/policystats-1',name='PolicyStatsTest',domain='policystats.com',recvDate='now')
return 0
finally:
purgeDispatcher.invoke('start',purgeHost)
preCalcSvc.invoke('start',purgeHost)
def setup(islandId):
sor = SOR()
hosts = sor.getHosts("purge-dispatcherV2")
if hosts is None or len(hosts) != 1:
print "Did not find expected host for purge-dispatcherV2", hosts
return 1
purgeHost = hosts.pop()
mc = ManagementContainer.getInstance()
purgeDispatcher = Service("purge-dispatcherV2")
preCalcSvc = Service("purge-precalc-service")
try:
purgeDispatcher.invoke("stop", purgeHost)
preCalcSvc.invoke("stop", purgeHost)
os.system(
"cd /tmp; /bin/rm -rf /tmp/corpus /tmp/purge-1; tar -xf endtoend-test-corpus.tar; mv /tmp/corpus /tmp/purge-1"
)
os.system("sed -i 's/@enron.com/@purge1.com/g' /tmp/purge-1/*")
setupCustomer(mc, islandId, corpus="/tmp/purge-1", name="purge1", domain="purge1.com", recvDate="now")
return 0
finally:
purgeDispatcher.invoke("start", purgeHost)
preCalcSvc.invoke("start", purgeHost)
caps = mc.getCustomerManager().getCustomerCapabilities(int(custId))
caps.setBooleanCapability(Capabilities.CAP_ALLOW_ARCHIVE_EDISCOVERY,True)
mc.getCustomerManager().saveCustomerCapabilities(caps)
caps = mc.getCustomerManager().getCustomerCapabilities(int(custId))
print custId,'ediscovery is',caps.getBooleanCapability(Capabilities.CAP_ALLOW_ARCHIVE_EDISCOVERY)
uName = '*****@*****.**'
token = getAuthToken(mc, custId, uName)
token2 = getAuthToken(mc, custId, '*****@*****.**')
token3 = getAuthToken(mc, custId, '*****@*****.**')
# get application server using SOR()
sor = SOR()
hosts = sor.getApplicationServers()
if hosts is None:
print 'Did not find expected host for Application Server ',hosts
sys.exit(1)
ApplicationServerHost = hosts.pop()
BaseURL = 'https://'+ApplicationServerHost+'/wfe/client/'
mainTest(token,token2,token3,BaseURL,mc,custId)
# # turn off ediscovery for customer
# caps = mc.getCustomerManager().getCustomerCapabilities(int(custId))
def test(islandId, numMessages, custName):
sor = SOR()
hosts = sor.getHosts("purge-dispatcherV2")
if hosts is None or len(hosts) != 1:
print "Did not find expected host for purge-dispatcherV2", hosts
sys.exit(1)
purgeHost = hosts.pop()
rc = True
mc = ManagementContainer.getInstance()
cm = mc.getCustomerManager()
amsm = mc.getActiveMailboxStoreManager()
purgeDispatcher = Service("purge-dispatcherV2")
preCalcSvc = Service("purge-precalc-service")
purgeDispatcher.invoke("stop", purgeHost)
preCalcSvc.invoke("stop", purgeHost)
custid = findCustomer(custName)
if custid < 0:
print "test failed because customer", custName, "was not found"
return False
prepDataForPurge(mc, custid, purgeHost)
preCalcSvc.invoke("start", purgeHost)
msgs = findMessages(mc, custid, numMessages)
msg = selectMessage(mc, msgs)
purgeMessage(mc, msg, custid)
partition = mc.getPartitionManager().getPartition(msg.getPartitionId())
tm = amsm.findMessages(
[
SearchConstraint(
IActiveMailboxStoreManager.PROP_CUST_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, int(custid)
),
SearchConstraint(
IActiveMailboxStoreManager.PROP_MSG_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, msg.getMessageId()
),
],
partition,
)
if not tm.isEmpty():
print "purge failed to delete message", msg
sys.exit(1)
for p in mc.getPartitionManager().listPartitions():
amsm.purgeRequest(p)
now = Date()
purgeDispatcher.invoke("start", purgeHost)
done = checkLastPurge(mc, now)
if done:
for msg in msgs:
partition = mc.getPartitionManager().getPartition(msg.getPartitionId())
tm = amsm.findMessages(
[
SearchConstraint(
IActiveMailboxStoreManager.PROP_CUST_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, int(custid)
),
SearchConstraint(
IActiveMailboxStoreManager.PROP_MSG_ID,
SearchConstraintOperator.CONSTRAINT_EQUALS,
msg.getMessageId(),
),
],
partition,
)
if not tm.isEmpty():
print "purge failed to delete", msg
rc = False
else:
rc = False
if rc is True:
print "purge successful"
else:
print "purge failed"
sys.exit(rc is True)
def test(islandId):
global purgeClient
global preCalcSvc
global purgeDispatcher
global purgeHost
sor = SOR()
hosts = sor.getHosts('purge-dispatcherV2')
if hosts is None or len(hosts) != 1:
print 'Did not find expected host for purge-dispatcherV2',hosts
sys.exit(1)
purgeHost = hosts.pop()
# ran into some shell issues. this apparently fixes it
os.system("cd")
rc = False
cm = mc.getCustomerManager()
purgeDispatcher = Service('purge-dispatcherV2')
preCalcSvc = Service('purge-precalc-service')
purgeClient = Service('purge-client')
debug("test: killing precalc and purge-dispatcher services on " + purgeHost + " and purge on local machine")
purgeDispatcher.invoke('stop',purgeHost)
preCalcSvc.invoke('stop',purgeHost)
purgeClient.invoke('stop',localhost)
if not os.path.exists(sentinelSpoolDir): os.makedirs(sentinelSpoolDir)
try:
debug("test: creating new customer")
# order of test cases (all have valid sentinel file in shared - do not change order!):
# valid batch in shared directory, completely corrupt copy in bigdisk
# valid batch in shared directory, no copy in bigdisk
# completely corrupted batch in shared directory, completely corrupt copy in bigdisk
# completely corrupted batch in shared directory, no copy in bigdisk
# no batch in shared directory, no copy in bigdisk
# valid batch in shared, no reading permissions copy in bigdisk
# partially valid batch in shared, partially valid copy in bigdisk
# partially valid batch in shared, completely corrupt copy in bigdisk
# partially valid batch in shared, no copy in bigdisk
# valid batch in shared directory, partially valid copy in bigdisk
#
# regression should be done in last test case!
rc = genericTest()
if rc: rc = genericTest(shouldDeleteSpoolBatch=True)
if rc: rc = genericTest(shouldMuckUpWorkBatch=True)
if rc: rc = genericTest(shouldMuckUpWorkBatch=True, shouldDeleteSpoolBatch=True)
if rc: rc = genericTest(shouldDeleteWorkBatch=True, shouldDeleteSpoolBatch=True, shouldMuckUpBatchFileInSpoolDir=False)
if rc: rc = genericTest(shouldChangePermissions=True)
if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldPartialSpoolBatch=True)
if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldMuckUpWorkBatch=True)
if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldDeleteSpoolBatch=True)
if rc: rc = genericTest(shouldPartialSpoolBatch=True)# , shouldDoRegression=True)
finally:
# clean up
cleanPurgeDirs()
debug("test: restarting purge dispatcher, purge, and precalc services")
purgeDispatcher.invoke('start',purgeHost)
preCalcSvc.invoke('start',purgeHost)
purgeClient.invoke('start',localhost)
debug("test: returning " + str(rc is not True))
log.close()
sys.exit(rc is not True)
def test(islandId,numMessages,custName):
sor = SOR()
hosts = sor.getHosts('purge-dispatcherV2')
if hosts is None or len(hosts) != 1:
print 'Did not find expected host for purge-dispatcherV2',hosts
sys.exit(1)
purgeHost = hosts.pop()
mc = ManagementContainer.getInstance()
cm = mc.getCustomerManager()
amsm = mc.getActiveMailboxStoreManager()
preCalcSvc = Service('purge-precalc-service')
preCalcSvc.invoke('stop',purgeHost)
custid = findCustomer(custName)
if custid < 0:
print 'test failed because customer',custName,'was not found'
return False
clearPolicyStats(custid)
rpm = mc.getRetentionPolicyManager()
rp = rpm.getLongestRetentionPolicy(custid)
pmm = mc.getPolicyMetaManager()
policyStats = pmm.getPolicyStats(custid)
if not policyStats.isEmpty():
print 'test failed because policy stats was not empty for customer',custid
return False
msgs = findMessages(mc,custid,numMessages)
for p in mc.getPartitionManager().listPartitions():
amsm.purgeRequest(p)
preCalcSvc.invoke('start',purgeHost)
# wait for the stats to get generated
maxTries = 60
while not areStatsGenerated(custid) and maxTries > 0:
print 'Waiting for pre-calc service to generate stats...'
time.sleep(10)
maxTries = maxTries - 1
if not areStatsGenerated(custid):
print 'test failed because policy stats were not generated within time alotted'
return False
os.system('ssh -o StrictHostKeyChecking=no %s "bash /opt/ems/bin/policy_stats"' % purgeHost)
policyStats = pmm.getPolicyStats(custid)
if policyStats.isEmpty():
print 'test failed because policy stats was empty for customer',custid
return False
if not policyStats.containsKey(rp.getId()):
print 'test failed becasue policy stats did not contain an entry for id',rp.getId()
return False
stat = policyStats.get(rp.getId())
print 'users',stat.getNumUsers(),'size',stat.getMessageSize(),'count',stat.getMessageCount()
if numMessages != stat.getMessageCount():
print 'test failed because expected',numMessages,'but only found',stat.getMessageCount(),'in stats'
return True
for dim in range(10, 100, 5):
a = np.random.rand(dim, dim)
A = (a + a.T) / 2
A = np.add(dim * np.identity(dim), A)
b = np.random.rand(dim)
print("Running Gaussian")
x_gauss, iteration_gauss, total_time_gauss, all_x_gauss, spectral_radius_gauss = gauss(
A, b, None, 1E-15, 1000000)
print("Running Jacobi")
x_jacobi, iteration_jacobi, total_time_jacobi, all_x_jacobi, spectral_radius_jacobi = jacobi(
A, b, None, 1E-15, 1000000)
print("Running Sor")
x_sor, iteration_sor, total_time_sor, all_x_sor, spectral_radius_sor = SOR(
A, b, None, 1E-15, 1000000, optimal_w(A))
iteration_d["gauss"].append(iteration_gauss)
iteration_d["jacobi"].append(iteration_jacobi)
iteration_d["sor"].append(iteration_sor)
time_d["gauss"].append(total_time_gauss)
time_d["jacobi"].append(total_time_jacobi)
time_d["sor"].append(total_time_sor)
spectral_d["gauss"].append(spectral_radius_gauss)
spectral_d["jacobi"].append(spectral_radius_jacobi)
spectral_d["sor"].append(spectral_radius_sor)
plt.figure(figsize=(10, 5))
plt.title("Iteration vs Size")
def runTests():
ok_count = 0
failed_count = 0
try:
mc = ManagementContainer.getInstance()
custList = mc.getCustomerManager().findCustomers([SearchConstraint(ICustomerManager.PROP_NAME, SearchConstraintOperator.CONSTRAINT_EQUALS, CUSTOMER_NAME)])
cust = custList[0]
custId = cust.getCustID()
# get application server using SOR()
#
# NOTE THAT YOU MUST INITIALIZE SOR(x) where x is the DOMAIN of the ems machine e.g. 'qa.messageone.com' if it is not an openstack backend
#
sor = SOR()
hosts = sor.getApplicationServers()
if hosts is None or len(hosts) == 0:
print 'Did not find expected host for Application Server ',hosts
sys.exit(1)
applicationServerHost = hosts.pop()
baseURL = 'https://'+applicationServerHost+'/wfe/client/'
token = getAuthToken(mc, custId, 'pallen' + '@' + CUSTOMER_DOMAIN)
# need to find a user that owns emails, and a folder in that association
print time.asctime(),'test without folder'
t0 = time.clock()
status, headers, content = searchCall(baseURL, token, '0 1 2 3 4 5 6 7 8 9 ', sort='0x0037:0', queryXML = '<SubRestriction></SubRestriction>',lang = None, folder = None)
t1 = time.clock()
print time.asctime(),'elapsed time =', (t1-t0), 'Sec'
print time.asctime(),len(content)
#print content
print time.asctime(),'test with folder'
t0 = time.clock()
status, headers, content = searchCall(baseURL, token, '0 1 2 3 4 5 6 7 8 9 ', sort='0x0037:0', queryXML = '<SubRestriction></SubRestriction>',lang = None, folder = 'allen-p/all_documents/')
t1 = time.clock()
print time.asctime(),'elapsed time =', (t1-t0), 'Sec'
print time.asctime(),len(content)
#print content
print time.asctime(),'test without folder'
t0 = time.clock()
status, headers, content = searchCall(baseURL, token, '0 1 2 3 4 5 6 7 8 9 ', sort='0x0037:0', queryXML = '<SubRestriction></SubRestriction>',lang = None, folder = None)
t1 = time.clock()
print time.asctime(),'elapsed time =', (t1-t0), 'Sec'
print time.asctime(),len(content)
#print content
print time.asctime(),'test with folder'
t0 = time.clock()
status, headers, content = searchCall(baseURL, token, '0 1 2 3 4 5 6 7 8 9 ', sort='0x0037:0', queryXML = '<SubRestriction></SubRestriction>',lang = None, folder = 'allen-p/all_documents/')
t1 = time.clock()
print time.asctime(),'elapsed time =', (t1-t0), 'Sec'
print time.asctime(),len(content)
print time.asctime(),'test without folder'
t0 = time.clock()
status, headers, content = searchCall(baseURL, token, '0 1 2 3 4 5 6 7 8 9 ', sort='0x0037:0', queryXML = '<SubRestriction></SubRestriction>',lang = None, folder = None)
t1 = time.clock()
print time.asctime(),'elapsed time =', (t1-t0), 'Sec'
print time.asctime(),len(content)
print time.asctime(),ok_count,'succeeded'
print time.asctime(),failed_count,'failures'
sys.exit(failed_count)
except Exception, e:
print time.asctime(),'error during test execution:', e
sys.exit(1)
def restartCache(cacheHost):
print time.asctime(),'restartCache host',cacheHost
maxAttempts = 5
while maxAttempts > 0:
os.system('ssh -o StrictHostKeyChecking=no %s "sh /opt/voltdb/restart-cache"' % cacheHost)
if checkCacheReady(cacheHost,sleepTime=5): return
maxAttempts = maxAttempts - 1
print time.asctime(),'Failed to restart cache to working state'
raise Exception('Cache failed to restart')
#
# main test body
#
if __name__ == '__main__':
sor = SOR()
print time.asctime(),'looking for all boxes:', sor.getAllManifestHosts()
print time.asctime(),'looking for cache boxes:', sor.findHostByName('cache-')
cacheHosts = sor.getHosts('voltdb')
if cacheHosts is None or len(cacheHosts)!=3:
print time.asctime(),'Did not find expected host for voltdb',cacheHosts
sys.exit(1)
cacheHost = cacheHosts[2]
dbHosts = sor.getHosts('postgresql')
if dbHosts is None or len(dbHosts) != 1:
print time.asctime(),'Did not find expected host for postgres',dhHosts
dbHost = dbHosts[0]
print time.asctime(),'Database host is ',dbHost
rc_1 = rc_2 = 1
