def test_compact_and_prune(self): ''' Replicates an instance where a statefile and another logfile with a lower iteration number can interact given a certain prune / compact cycle ''' def createNewHarnessAndView(): self.harness = self.newHarness(maxLogFileSizeMb=.08) self.view = self.harness.newView() self.harness.subscribeToKeyspace(self.view, self.keyspaceName) return self.harness, self.view self.harness.teardown() self.harness, self.view = createNewHarnessAndView() totalSize = 0 self.harness.sendPingAndCompact() # ensure we have a logfile started here... while getLogFileLen(self.tempDir, self.keyspaceName) < 1024 * 100: for ix in range(200): self.harness.writeToKeyspace(self.view, self.keyspaceName, NativeJson.Json(self.randomString()), NativeJson.Json(self.randomString())) self.harness.manager.check() LogFilePruner.pruneLogFiles(self.tempDir) self.harness.teardown() self.harness, self.view = createNewHarnessAndView() loopAgain = True while loopAgain: for ix in range(200): self.harness.writeToKeyspace(self.view, self.keyspaceName, NativeJson.Json(self.randomString()), NativeJson.Json(self.randomString())) loopAgain = getLogFileLen(self.tempDir, self.keyspaceName) < 1024 * 100 self.harness.manager.check()
def startService(self): if self.cachePath != '': # Compress all keyspaces before starting to accept connections. # This increases the loading speed of keyspaces self.compressOrphandLogFiles() LogFilePruner.pruneLogFiles(self.cachePath) self.socketServerThread.start() self.logfilePruneThread.start()
def logFilePruner(self): if self.cachePath == '': return logging.info("Starting log-file pruning loop") while not self.stoppedFlag.is_set(): try: LogFilePruner.pruneLogFiles(self.cachePath) except: # We don't want to stop pruning just because there was an # error logging.error("Error pruning log files in %s\n%s", self.cachePath, traceback.format_exc()) t = time.time() while not self.stoppedFlag.is_set() and \ time.time() - t < Setup.config().sharedStateLogPruneFrequency: time.sleep(1)
def logFilePruner(self): if self.cachePath == '': return logging.info("Starting log-file pruning loop") while not self.stoppedFlag.is_set(): try: LogFilePruner.pruneLogFiles(self.cachePath) except: # We don't want to stop pruning just because there was an # error logging.error("Error pruning log files in %s\n%s", self.cachePath, traceback.format_exc()) t = time.time() while not self.stoppedFlag.is_set() and \ time.time() - t < Setup.config().sharedStateLogPruneIntervalSeconds: time.sleep(1)
def disabled_logfile_prune(self): items = self.writeDataUntilNStatefilesExist(10, self.randomString, self.randomString) allStateFiles = sorted(getStateFiles(self.tempDir, self.keyspaceName)) filesToCorrupt = allStateFiles[-5:] for fileToCorrupt in filesToCorrupt: with open(fileToCorrupt, 'a') as f: size = os.stat(fileToCorrupt).st_size startByte = int(0) bytesToCorrupt = min(1024 * 1024, size - startByte) f.seek(startByte) f.write(self.randomString(bytesToCorrupt).toSimple()) self.harness.teardown() numStateFilesBeforePrune = len(getStateFiles(self.tempDir, self.keyspaceName)) LogFilePruner.pruneLogFiles(self.tempDir) self.assertLess(len(getStateFiles(self.tempDir, self.keyspaceName)), numStateFilesBeforePrune) self.harness = self.newHarness() view = self.harness.newView() self.harness.subscribeToKeyspace(view, self.keyspaceName) allItemsFromView = self.harness.getAllItemsFromView(view, self.keyspaceName) view = self.harness.newView() self.harness.subscribeToKeyspace(view, self.keyspaceName) allItemsFromView = self.harness.getAllItemsFromView(view, self.keyspaceName) allKeysInView = set(x[0][0] for x in allItemsFromView) self.assertEqual(len(set(items.keys())), len(allKeysInView)) self.assertEqual(set(items.keys()), allKeysInView) self.assertEqual(set(items.items()), set((x[0][0], x[1]) for x in allItemsFromView))