def test_compact_and_prune(self):
'''
Replicates an instance where a statefile and another logfile with a lower iteration number
can interact given a certain prune / compact cycle
'''
def createNewHarnessAndView():
self.harness = self.newHarness(maxLogFileSizeMb=.08)
self.view = self.harness.newView()
self.harness.subscribeToKeyspace(self.view, self.keyspaceName)
return self.harness, self.view
self.harness.teardown()
self.harness, self.view = createNewHarnessAndView()
totalSize = 0
self.harness.sendPingAndCompact()
# ensure we have a logfile started here...
while getLogFileLen(self.tempDir, self.keyspaceName) < 1024 * 100:
for ix in range(200):
self.harness.writeToKeyspace(self.view, self.keyspaceName, NativeJson.Json(self.randomString()), NativeJson.Json(self.randomString()))
self.harness.manager.check()
LogFilePruner.pruneLogFiles(self.tempDir)
self.harness.teardown()
self.harness, self.view = createNewHarnessAndView()
loopAgain = True
while loopAgain:
for ix in range(200):
self.harness.writeToKeyspace(self.view, self.keyspaceName, NativeJson.Json(self.randomString()), NativeJson.Json(self.randomString()))
loopAgain = getLogFileLen(self.tempDir, self.keyspaceName) < 1024 * 100
self.harness.manager.check()
def startService(self):
if self.cachePath != '':
# Compress all keyspaces before starting to accept connections.
# This increases the loading speed of keyspaces
self.compressOrphandLogFiles()
LogFilePruner.pruneLogFiles(self.cachePath)
self.socketServerThread.start()
self.logfilePruneThread.start()
def startService(self):
if self.cachePath != '':
# Compress all keyspaces before starting to accept connections.
# This increases the loading speed of keyspaces
self.compressOrphandLogFiles()
LogFilePruner.pruneLogFiles(self.cachePath)
self.socketServerThread.start()
self.logfilePruneThread.start()
def logFilePruner(self):
if self.cachePath == '':
return
logging.info("Starting log-file pruning loop")
while not self.stoppedFlag.is_set():
try:
LogFilePruner.pruneLogFiles(self.cachePath)
except:
# We don't want to stop pruning just because there was an
# error
logging.error("Error pruning log files in %s\n%s",
self.cachePath, traceback.format_exc())
t = time.time()
while not self.stoppedFlag.is_set() and \
time.time() - t < Setup.config().sharedStateLogPruneFrequency:
time.sleep(1)
def logFilePruner(self):
if self.cachePath == '':
return
logging.info("Starting log-file pruning loop")
while not self.stoppedFlag.is_set():
try:
LogFilePruner.pruneLogFiles(self.cachePath)
except:
# We don't want to stop pruning just because there was an
# error
logging.error("Error pruning log files in %s\n%s",
self.cachePath,
traceback.format_exc())
t = time.time()
while not self.stoppedFlag.is_set() and \
time.time() - t < Setup.config().sharedStateLogPruneIntervalSeconds:
time.sleep(1)
def disabled_logfile_prune(self):
items = self.writeDataUntilNStatefilesExist(10, self.randomString, self.randomString)
allStateFiles = sorted(getStateFiles(self.tempDir, self.keyspaceName))
filesToCorrupt = allStateFiles[-5:]
for fileToCorrupt in filesToCorrupt:
with open(fileToCorrupt, 'a') as f:
size = os.stat(fileToCorrupt).st_size
startByte = int(0)
bytesToCorrupt = min(1024 * 1024, size - startByte)
f.seek(startByte)
f.write(self.randomString(bytesToCorrupt).toSimple())
self.harness.teardown()
numStateFilesBeforePrune = len(getStateFiles(self.tempDir, self.keyspaceName))
LogFilePruner.pruneLogFiles(self.tempDir)
self.assertLess(len(getStateFiles(self.tempDir, self.keyspaceName)),
numStateFilesBeforePrune)
self.harness = self.newHarness()
view = self.harness.newView()
self.harness.subscribeToKeyspace(view, self.keyspaceName)
allItemsFromView = self.harness.getAllItemsFromView(view, self.keyspaceName)
view = self.harness.newView()
self.harness.subscribeToKeyspace(view, self.keyspaceName)
allItemsFromView = self.harness.getAllItemsFromView(view, self.keyspaceName)
allKeysInView = set(x[0][0] for x in allItemsFromView)
self.assertEqual(len(set(items.keys())), len(allKeysInView))
self.assertEqual(set(items.keys()), allKeysInView)
self.assertEqual(set(items.items()), set((x[0][0], x[1]) for x in allItemsFromView))
