Esempio n. 1
0
def setup(islandId):
    sor = SOR()
    hosts = sor.getHosts('purge-dispatcherV2')
    if hosts is None or len(hosts) != 1:
        print 'Did not find expected host for purge-dispatcherV2',hosts
        return 1
    purgeHost = hosts.pop()

    mc = ManagementContainer.getInstance()
    purgeDispatcher = Service('purge-dispatcherV2')
    preCalcSvc = Service('purge-precalc-service')
    
    try:
        purgeDispatcher.invoke('stop',purgeHost)
        preCalcSvc.invoke('stop',purgeHost)

        os.system("cd /tmp; /bin/rm -rf /tmp/corpus /tmp/policystats-1; tar -xf endtoend-test-corpus.tar; mv /tmp/corpus /tmp/policystats-1");
        os.system("sed -i 's/@enron.com/@policystats.com/g' /tmp/policystats-1/*");
        
        setupCustomer(mc,islandId,corpus='/tmp/policystats-1',name='PolicyStatsTest',domain='policystats.com',recvDate='now')
        
        return 0
    finally:
        purgeDispatcher.invoke('start',purgeHost)
        preCalcSvc.invoke('start',purgeHost)
Esempio n. 2
0
def setup(islandId):
    sor = SOR()
    hosts = sor.getHosts("purge-dispatcherV2")
    if hosts is None or len(hosts) != 1:
        print "Did not find expected host for purge-dispatcherV2", hosts
        return 1
    purgeHost = hosts.pop()

    mc = ManagementContainer.getInstance()
    purgeDispatcher = Service("purge-dispatcherV2")
    preCalcSvc = Service("purge-precalc-service")

    try:
        purgeDispatcher.invoke("stop", purgeHost)
        preCalcSvc.invoke("stop", purgeHost)

        os.system(
            "cd /tmp; /bin/rm -rf /tmp/corpus /tmp/purge-1; tar -xf endtoend-test-corpus.tar; mv /tmp/corpus /tmp/purge-1"
        )
        os.system("sed -i 's/@enron.com/@purge1.com/g' /tmp/purge-1/*")

        setupCustomer(mc, islandId, corpus="/tmp/purge-1", name="purge1", domain="purge1.com", recvDate="now")

        return 0
    finally:
        purgeDispatcher.invoke("start", purgeHost)
        preCalcSvc.invoke("start", purgeHost)
Esempio n. 3
0
        caps = mc.getCustomerManager().getCustomerCapabilities(int(custId))
        caps.setBooleanCapability(Capabilities.CAP_ALLOW_ARCHIVE_EDISCOVERY,True)
        mc.getCustomerManager().saveCustomerCapabilities(caps)
        caps = mc.getCustomerManager().getCustomerCapabilities(int(custId))
        print custId,'ediscovery is',caps.getBooleanCapability(Capabilities.CAP_ALLOW_ARCHIVE_EDISCOVERY)


        uName = '*****@*****.**'

        token = getAuthToken(mc, custId, uName)
        token2 = getAuthToken(mc, custId, '*****@*****.**')
        token3 = getAuthToken(mc, custId, '*****@*****.**')


        # get application server using SOR()
        sor = SOR()
        hosts = sor.getApplicationServers()

        if hosts is None:
            print 'Did not find expected host for Application Server ',hosts
            sys.exit(1)
        ApplicationServerHost = hosts.pop()

        BaseURL = 'https://'+ApplicationServerHost+'/wfe/client/'



        mainTest(token,token2,token3,BaseURL,mc,custId)

#        # turn off ediscovery for customer
#        caps = mc.getCustomerManager().getCustomerCapabilities(int(custId))
Esempio n. 4
0
def test(islandId, numMessages, custName):
    sor = SOR()
    hosts = sor.getHosts("purge-dispatcherV2")
    if hosts is None or len(hosts) != 1:
        print "Did not find expected host for purge-dispatcherV2", hosts
        sys.exit(1)
    purgeHost = hosts.pop()

    rc = True
    mc = ManagementContainer.getInstance()
    cm = mc.getCustomerManager()
    amsm = mc.getActiveMailboxStoreManager()
    purgeDispatcher = Service("purge-dispatcherV2")
    preCalcSvc = Service("purge-precalc-service")

    purgeDispatcher.invoke("stop", purgeHost)
    preCalcSvc.invoke("stop", purgeHost)

    custid = findCustomer(custName)
    if custid < 0:
        print "test failed because customer", custName, "was not found"
        return False

    prepDataForPurge(mc, custid, purgeHost)

    preCalcSvc.invoke("start", purgeHost)

    msgs = findMessages(mc, custid, numMessages)

    msg = selectMessage(mc, msgs)

    purgeMessage(mc, msg, custid)

    partition = mc.getPartitionManager().getPartition(msg.getPartitionId())

    tm = amsm.findMessages(
        [
            SearchConstraint(
                IActiveMailboxStoreManager.PROP_CUST_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, int(custid)
            ),
            SearchConstraint(
                IActiveMailboxStoreManager.PROP_MSG_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, msg.getMessageId()
            ),
        ],
        partition,
    )

    if not tm.isEmpty():
        print "purge failed to delete message", msg
        sys.exit(1)

    for p in mc.getPartitionManager().listPartitions():
        amsm.purgeRequest(p)

    now = Date()

    purgeDispatcher.invoke("start", purgeHost)

    done = checkLastPurge(mc, now)

    if done:
        for msg in msgs:
            partition = mc.getPartitionManager().getPartition(msg.getPartitionId())
            tm = amsm.findMessages(
                [
                    SearchConstraint(
                        IActiveMailboxStoreManager.PROP_CUST_ID, SearchConstraintOperator.CONSTRAINT_EQUALS, int(custid)
                    ),
                    SearchConstraint(
                        IActiveMailboxStoreManager.PROP_MSG_ID,
                        SearchConstraintOperator.CONSTRAINT_EQUALS,
                        msg.getMessageId(),
                    ),
                ],
                partition,
            )
            if not tm.isEmpty():
                print "purge failed to delete", msg
                rc = False
    else:
        rc = False

    if rc is True:
        print "purge successful"
    else:
        print "purge failed"

    sys.exit(rc is True)
Esempio n. 5
0
def test(islandId):
  global purgeClient
  global preCalcSvc
  global purgeDispatcher
  global purgeHost
  
  sor = SOR()
  hosts = sor.getHosts('purge-dispatcherV2')
  if hosts is None or len(hosts) != 1:
    print 'Did not find expected host for purge-dispatcherV2',hosts
    sys.exit(1)
  purgeHost = hosts.pop()

  # ran into some shell issues. this apparently fixes it
  os.system("cd")

  rc = False
  cm = mc.getCustomerManager()
  purgeDispatcher = Service('purge-dispatcherV2')
  preCalcSvc = Service('purge-precalc-service')
  purgeClient = Service('purge-client')

  debug("test: killing precalc and purge-dispatcher services on " + purgeHost + " and purge on local machine")
  purgeDispatcher.invoke('stop',purgeHost)
  preCalcSvc.invoke('stop',purgeHost)
  purgeClient.invoke('stop',localhost)
  if not os.path.exists(sentinelSpoolDir): os.makedirs(sentinelSpoolDir)

  try:
    debug("test: creating new customer")

    # order of test cases (all have valid sentinel file in shared - do not change order!):
    #   valid batch in shared directory, completely corrupt copy in bigdisk
    #   valid batch in shared directory, no copy in bigdisk
    #   completely corrupted batch in shared directory, completely corrupt copy in bigdisk
    #   completely corrupted batch in shared directory, no copy in bigdisk
    #   no batch in shared directory, no copy in bigdisk
    #   valid batch in shared, no reading permissions copy in bigdisk
    #   partially valid batch in shared, partially valid copy in bigdisk
    #   partially valid batch in shared, completely corrupt copy in bigdisk
    #   partially valid batch in shared, no copy in bigdisk
    #   valid batch in shared directory, partially valid copy in bigdisk
    #
    #   regression should be done in last test case!


    rc = genericTest()
    if rc: rc = genericTest(shouldDeleteSpoolBatch=True)
    if rc: rc = genericTest(shouldMuckUpWorkBatch=True)
    if rc: rc = genericTest(shouldMuckUpWorkBatch=True, shouldDeleteSpoolBatch=True)
    if rc: rc = genericTest(shouldDeleteWorkBatch=True, shouldDeleteSpoolBatch=True, shouldMuckUpBatchFileInSpoolDir=False)
    if rc: rc = genericTest(shouldChangePermissions=True)
    if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldPartialSpoolBatch=True)
    if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldMuckUpWorkBatch=True)
    if rc: rc = genericTest(shouldPartialWorkBatch=True, shouldDeleteSpoolBatch=True)
    if rc: rc = genericTest(shouldPartialSpoolBatch=True)# , shouldDoRegression=True)
        
  finally:
    # clean up
    cleanPurgeDirs()
    debug("test: restarting purge dispatcher, purge, and precalc services")
    purgeDispatcher.invoke('start',purgeHost)
    preCalcSvc.invoke('start',purgeHost)
    purgeClient.invoke('start',localhost)

  debug("test: returning " + str(rc is not True))
  log.close()
  sys.exit(rc is not True)
Esempio n. 6
0
def test(islandId,numMessages,custName):
    sor = SOR()
    hosts = sor.getHosts('purge-dispatcherV2')
    if hosts is None or len(hosts) != 1:
        print 'Did not find expected host for purge-dispatcherV2',hosts
        sys.exit(1)
    purgeHost = hosts.pop()

    mc = ManagementContainer.getInstance()
    cm = mc.getCustomerManager()
    amsm = mc.getActiveMailboxStoreManager()
    preCalcSvc = Service('purge-precalc-service')
    
    preCalcSvc.invoke('stop',purgeHost)
    
    custid = findCustomer(custName)
    if custid < 0:
        print 'test failed because customer',custName,'was not found'
        return False

    clearPolicyStats(custid)

    rpm = mc.getRetentionPolicyManager()
    rp = rpm.getLongestRetentionPolicy(custid)
    pmm = mc.getPolicyMetaManager()

    policyStats = pmm.getPolicyStats(custid) 

    if not policyStats.isEmpty():
        print 'test failed because policy stats was not empty for customer',custid
        return False

    msgs = findMessages(mc,custid,numMessages)
        
    for p in mc.getPartitionManager().listPartitions():
        amsm.purgeRequest(p)

    preCalcSvc.invoke('start',purgeHost)

    # wait for the stats to get generated
    maxTries = 60
    while not areStatsGenerated(custid) and maxTries > 0:
        print 'Waiting for pre-calc service to generate stats...'
        time.sleep(10)
        maxTries = maxTries - 1

    if not areStatsGenerated(custid):
        print 'test failed because policy stats were not generated within time alotted'
        return False

    os.system('ssh -o StrictHostKeyChecking=no %s "bash /opt/ems/bin/policy_stats"' % purgeHost)

    policyStats = pmm.getPolicyStats(custid) 

    if policyStats.isEmpty():
        print 'test failed because policy stats was empty for customer',custid
        return False

    if not policyStats.containsKey(rp.getId()):
        print 'test failed becasue policy stats did not contain an entry for id',rp.getId()
        return False

    stat = policyStats.get(rp.getId())

    print 'users',stat.getNumUsers(),'size',stat.getMessageSize(),'count',stat.getMessageCount()
    
    if numMessages != stat.getMessageCount():
        print 'test failed because expected',numMessages,'but only found',stat.getMessageCount(),'in stats'

    return True
Esempio n. 7
0
    for dim in range(10, 100, 5):

        a = np.random.rand(dim, dim)
        A = (a + a.T) / 2
        A = np.add(dim * np.identity(dim), A)

        b = np.random.rand(dim)

        print("Running Gaussian")
        x_gauss, iteration_gauss, total_time_gauss, all_x_gauss, spectral_radius_gauss = gauss(
            A, b, None, 1E-15, 1000000)
        print("Running Jacobi")
        x_jacobi, iteration_jacobi, total_time_jacobi, all_x_jacobi, spectral_radius_jacobi = jacobi(
            A, b, None, 1E-15, 1000000)
        print("Running Sor")
        x_sor, iteration_sor, total_time_sor, all_x_sor, spectral_radius_sor = SOR(
            A, b, None, 1E-15, 1000000, optimal_w(A))

        iteration_d["gauss"].append(iteration_gauss)
        iteration_d["jacobi"].append(iteration_jacobi)
        iteration_d["sor"].append(iteration_sor)

        time_d["gauss"].append(total_time_gauss)
        time_d["jacobi"].append(total_time_jacobi)
        time_d["sor"].append(total_time_sor)

        spectral_d["gauss"].append(spectral_radius_gauss)
        spectral_d["jacobi"].append(spectral_radius_jacobi)
        spectral_d["sor"].append(spectral_radius_sor)

    plt.figure(figsize=(10, 5))
    plt.title("Iteration vs Size")
Esempio n. 8
0
def runTests():
    ok_count = 0
    failed_count = 0

    try:
        mc = ManagementContainer.getInstance()
        custList = mc.getCustomerManager().findCustomers([SearchConstraint(ICustomerManager.PROP_NAME, SearchConstraintOperator.CONSTRAINT_EQUALS, CUSTOMER_NAME)])
        cust = custList[0]
        custId = cust.getCustID()

        # get application server using SOR()
        #
        # NOTE THAT YOU MUST INITIALIZE SOR(x) where x is the DOMAIN of the ems machine e.g. 'qa.messageone.com' if it is not an openstack backend
        #
        sor = SOR()
        hosts = sor.getApplicationServers()

        if hosts is None or len(hosts) == 0:
            print 'Did not find expected host for Application Server ',hosts
            sys.exit(1)
        applicationServerHost = hosts.pop()

        baseURL = 'https://'+applicationServerHost+'/wfe/client/'

        token = getAuthToken(mc, custId, 'pallen' + '@' + CUSTOMER_DOMAIN)

        # need to find a user that owns emails, and a folder in that association

        print time.asctime(),'test without folder'
        t0 = time.clock()
        status, headers, content = searchCall(baseURL, token, '0 1 2 3 4 5 6 7 8 9 ', sort='0x0037:0', queryXML = '<SubRestriction></SubRestriction>',lang = None, folder = None)
        t1 = time.clock()
        print time.asctime(),'elapsed time =', (t1-t0), 'Sec'
        print time.asctime(),len(content)
        #print content

        print time.asctime(),'test with folder'
        t0 = time.clock()
        status, headers, content = searchCall(baseURL, token, '0 1 2 3 4 5 6 7 8 9 ', sort='0x0037:0', queryXML = '<SubRestriction></SubRestriction>',lang = None, folder = 'allen-p/all_documents/')
        t1 = time.clock()
        print time.asctime(),'elapsed time =', (t1-t0), 'Sec'
        print time.asctime(),len(content)
        #print content

        print time.asctime(),'test without folder'
        t0 = time.clock()
        status, headers, content = searchCall(baseURL, token, '0 1 2 3 4 5 6 7 8 9 ', sort='0x0037:0', queryXML = '<SubRestriction></SubRestriction>',lang = None, folder = None)
        t1 = time.clock()
        print time.asctime(),'elapsed time =', (t1-t0), 'Sec'
        print time.asctime(),len(content)
        #print content

        print time.asctime(),'test with folder'
        t0 = time.clock()
        status, headers, content = searchCall(baseURL, token, '0 1 2 3 4 5 6 7 8 9 ', sort='0x0037:0', queryXML = '<SubRestriction></SubRestriction>',lang = None, folder = 'allen-p/all_documents/')
        t1 = time.clock()
        print time.asctime(),'elapsed time =', (t1-t0), 'Sec'
        print time.asctime(),len(content)

        print time.asctime(),'test without folder'
        t0 = time.clock()
        status, headers, content = searchCall(baseURL, token, '0 1 2 3 4 5 6 7 8 9 ', sort='0x0037:0', queryXML = '<SubRestriction></SubRestriction>',lang = None, folder = None)
        t1 = time.clock()
        print time.asctime(),'elapsed time =', (t1-t0), 'Sec'
        print time.asctime(),len(content)

        print time.asctime(),ok_count,'succeeded'
        print time.asctime(),failed_count,'failures'

        sys.exit(failed_count)

    except Exception, e:
        print time.asctime(),'error during test execution:', e
        sys.exit(1)
Esempio n. 9
0
def restartCache(cacheHost):
    print time.asctime(),'restartCache host',cacheHost
    maxAttempts = 5
    while maxAttempts > 0:
        os.system('ssh -o StrictHostKeyChecking=no %s "sh /opt/voltdb/restart-cache"' % cacheHost)
        if checkCacheReady(cacheHost,sleepTime=5): return
        maxAttempts = maxAttempts - 1

    print time.asctime(),'Failed to restart cache to working state'
    raise Exception('Cache failed to restart')
#
# main test body
#
if __name__ == '__main__':
    
    sor = SOR()
    print time.asctime(),'looking for all boxes:', sor.getAllManifestHosts()
    print time.asctime(),'looking for cache boxes:', sor.findHostByName('cache-')
    cacheHosts = sor.getHosts('voltdb')
    if cacheHosts is None or len(cacheHosts)!=3:
        print time.asctime(),'Did not find expected host for voltdb',cacheHosts
        sys.exit(1)
    cacheHost = cacheHosts[2]

    dbHosts = sor.getHosts('postgresql')
    if dbHosts is None or len(dbHosts) != 1:
        print time.asctime(),'Did not find expected host for postgres',dhHosts
    dbHost = dbHosts[0]
    print time.asctime(),'Database host is ',dbHost

    rc_1 = rc_2 = 1