def migrate(custIds,islandId,sourceIslandId,minRange,maxRange,showAndExitOnly=False):
global QUEUE,MsgsProcessed,StartCount,TotSearchTime
mc = ManagementContainer.getInstance()
ism = mc.getIndexSearchManager()
im = mc.getIslandManager()
sourceIsland = im.getIsland(int(sourceIslandId))
sourceType = sourceIsland.getIndexPlatformVersion()
# search for documents
isc = IndexSearchConstraint(None,None)
isc.sortBy('storageid',True)
isc.constrainByMinimumStorageId(minRange)
isc.constrainByIsland(mc.getIslandManager().getIsland(sourceIslandId))
if IndexPlatformVersion.SOLR_x == sourceType :
query = buildSolrQuery(custIds, maxRange)
else:
query = buildFastQuery(custIds, maxRange)
start = time.time()
sr = search(query,isc,ism)
StartCount = sr.getDocCount()
elapsed = time.time() - start
TotSearchTime = TotSearchTime + elapsed
log('PERF: search',StartCount,'msgs, time',elapsed,'s','total',TotSearchTime,'s')
if showAndExitOnly:
return
hitsRemaining = ism.getMaxIndextHits()
lastStorageId = 0
iter = sr.documents().iterator()
while checkSearchIter(iter):
doc = iter.next()
hitsRemaining -= 1
storageId = doc.getStorageID()
# only process each EMS message ID one time
if storageId != lastStorageId:
QUEUE.put(doc)
lastStorageId = storageId
if hitsRemaining <= 0 or not checkSearchIter(iter):
log('STAT: getting new search results beyond storageId',storageId)
isc.constrainByMinimumStorageId(storageId)
start = time.time()
sr = search(query + ' and isattachment:0',isc,ism)
elapsed = time.time() - start
TotSearchTime = TotSearchTime + elapsed
log('PERF: search time',elapsed,'s','total',TotSearchTime,'s')
hitsRemaining = ism.getMaxIndextHits()
iter = sr.documents().iterator()
log('STAT: search work complete')
def test(islandId, numMessages, esHosts):
mc = ManagementContainer.getInstance()
lastVal = None
custId = None
service = Service("storageimporter")
msgs = None
try:
lastVal, nextVal = setMessageId(mc, Integer.MAX_VALUE - 2)
print "lastVal,nextVal", lastVal, nextVal
print "restart storageimporters", esHosts
service.invoke("restart", esHosts)
custId = setupCustomer(mc, islandId)
msgs = findMessages(mc, custId, numMessages)
print "created messages", msgs
checkSearchStatus(mc, msgs, custId)
ism = mc.getIndexSearchManager()
isc = IndexSearchConstraint(None, None)
isc.sortBy("storageid", True)
isc.constrainByMinimumStorageId(0)
isc.constrainByIsland(mc.getIslandManager().getIsland(islandId))
print "DBG min", 0
query = "customerid:" + str(custId)
sr = ism.searchAndWrap(query, isc, CallerApp.INTERNAL)
if sr.getDocCount() != 10:
print >> sys.stderr, "Failed to find 10 documents when minStorageId = 0, found", sr.getDocCount()
return 1
print "DBG min", Integer.MAX_VALUE - 1
isc.constrainByMinimumStorageId(Integer.MAX_VALUE - 1)
sr = ism.searchAndWrap(query, isc, CallerApp.INTERNAL)
if sr.getDocCount() != 9:
print >> sys.stderr, "Failed to find 9 documents when minStorageId = ", Integer.MAX_VALUE - 1, ",found", sr.getDocCount()
return 1
print "DBG min", Integer.MAX_VALUE + 1
isc.constrainByMinimumStorageId(Integer.MAX_VALUE + 1)
sr = ism.searchAndWrap(query, isc, CallerApp.INTERNAL)
if sr.getDocCount() != 7:
print >> sys.stderr, "Failed to find 7 documents when minStorageId = ", Integer.MAX_VALUE + 1, ",found", sr.getDocCount()
return 1
finally:
if msgs is not None:
for msg in msgs:
purgeMessage(mc, msg, custId)
if custId is not None:
mc.getCustomerManager().deleteCustomers([custId])
if lastVal is not None:
lastVal, nextVal = setMessageId(mc, lastVal)
print "lastVal,nextVal", lastVal, nextVal
print "restart storageimporters", esHosts
service.invoke("restart", esHosts)
return 0
def search(isle, term):
mc = ManagementContainer.getInstance()
sm = mc.getIndexSearchManager()
im = mc.getIslandManager()
pm = mc.getPartitionManager()
fIS = IndexSearchConstraint(None, None)
fIS.constrainByIsland(isle)
fIS.constrainByNumberOfHitsToReturn(10)
fIS.constrainByMinimumStorageId(0)
fIS.setOutputFields(["storageid", "partitionid"])
fIS.sortBy("storageid", True)
done = False
cnt = 0
lastID = 0
while not done:
ok = False
while not ok:
try:
fQR = sm.search(term, fIS, None, CallerApp.INTERNAL)
ok = True
except Throwable, t:
print "Exception caught during search, retry = true", t
t.printStackTrace()
numDocs = fQR.getDocCount()
print "Found numDocs", numDocs
done = fQR.getDocCount() == 0
print "Preview some data"
for doc in fQR.documents():
# print doc.getPartitionID(),pm.getPartition(doc.getPartitionID()).isReadOnly(),doc.getStorageID(),doc.getReceivedDate()
print doc.getPartitionID(), doc.getStorageID()
lastID = doc.getStorageID()
cnt += 1
fIS.constrainByMinimumStorageId(lastID)
# just loop once
done = True
