def getMsgRaw(fileName, rs): msgID = rs.getLong(1) size = rs.getInt(2) storageLocation = StoredMessage.stripOldStorageDir(rs.getString(3)) custID = rs.getInt(4) ts = rs.getTimestamp(5) receivedDate = None if ts is not None: receivedDate = Date(ts.getTime) msg = StoredMessage(custID, msgID, storageLocation, None, None, receivedDate, 0, 0, 0, 0, None, amPartId, size, size, None, None) if msg: # do some validation if long(msgId) != msg.getMessageId(): printQueue.append("consumer: ERROR: message ID " + str(msgId) + " not the same as " + str(msg.getMessageId())) #print "found message", msgId, "in DB for AM partition" msgStream = msg.getEncryptedCompressedContentStream() if msgStream: out = FileOutputStream(fileName) FileUtils.copyStream(msg.getEncryptedCompressedContentStream(), out) out.flush() out.close() #print "found message", msgId, "on disk in AM partition" return True else: printErrorToRemigrate(msgId, " not found on disk for AM partition.") return False else: printErrorToRemigrate(msgId, " not found in db for AM partition.") return False
# next, see if they now match. if they do, get to work. if not, throw an error if rs.getLong(1) == long(str.strip(line)): # build up a StoredMessage msgID = rs.getLong(1) size = rs.getInt(2) storageLocation = StoredMessage.stripOldStorageDir(rs.getString(3)) custID = rs.getInt(4) ts = rs.getTimestamp(5) receivedDate = None if ts is not None: receivedDate = Date(ts.getTime()) msg = StoredMessage(custID, msgID, storageLocation, None, None, receivedDate, 0, 0, 0, 0, None, amPartId, size, size, None, None) # build up a batch if it isn't full yet if runningCount < batchSize: printQueue.add("debug(main): adding msg " + str(msg.getMessageId()) + " to a batch") queueToBuild.append(msg) runningCount += 1 # otherwise we're ready to push to a thread and start with a new batch else: # keep on trying to enqueue the current batch into an empty thread # if this one doesn't work, increment the index until we find an available one old_index = index while not threadList[index].canAddToQueue(): index += 1 index %= numThreads if index == old_index: time.sleep(sleepTime) # avoid CPU thrash after we've looped over all of the threads # do the actual push to the thread