def setUp(self): self.sharedState = SharedStateTestHarness(inMemory = True) self.keyspacePath = (0, 1) self.keyspaceName = Node.keyspacePathToKeyspaceName(self.keyspacePath) self.keyName = 'key' self.keyValue = 'value' self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", self.keyspaceName, 1) self.keyRange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False)
def setUp(self): self.sharedState = SharedStateTestHarness(inMemory=True) self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer() self.synchronizer.attachView(self.sharedState.newView()) self.synchronizer.__enter__()
class TestPersistentCacheIndex(unittest.TestCase): def setUp(self): self.sharedState = SharedStateTestHarness(inMemory=True) self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer() self.synchronizer.attachView(self.sharedState.newView()) self.synchronizer.__enter__() def tearDown(self): self.synchronizer.__exit__(None, None, None) def waitForSync(self, predicate, *synchronizers): if not synchronizers: synchronizers = [self.synchronizer] ComputedGraph.assertHasGraph() passes = 0 while passes < 100: for s in synchronizers: s.update() if predicate(): break time.sleep(.1) passes += 1 self.assertTrue(predicate()) @ComputedGraphTestHarness.UnderHarness def test_persistentCacheUnderLoad(self): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler ) t0 = time.time() #add 100k pages, which is enough for about 5 TB of data for index in range(100000): if index % 1000 == 0 and index > 0: print index, (time.time() - t0) / (index / 1000.0), " seconds per 1000" cppView1.addPage(sha1("page" + str(index)), HashSet(), 1, sha1("")) print "took ", time.time() - t0, " to add 100k." t1 = time.time() bytes0 = TCMallocNative.getBytesUsed() cppView2 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler ) while cppView2.totalBytesInCache() < 100000: time.sleep(.1) print cppView2.totalBytesInCache() print "took ", time.time() - t1, " to load 100k. Total RAM is ", (TCMallocNative.getBytesUsed() - bytes0) / 1024 / 1024.0, " MB per view"\ @ComputedGraphTestHarness.UnderHarness def test_circularPageReferencesAreInvalid(self): self.circularPageReferenceTest(True) @ComputedGraphTestHarness.UnderHarness def test_noncircularPageReferencesAreValid(self): self.circularPageReferenceTest(False) def circularPageReferenceTest(self, shouldBeInvalid): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler ) computationId = CumulusNative.ComputationId.Root( sha1("computation") ) checkpointRequest = CumulusNative.CheckpointRequest(0.0, True, computationId) cppView1.addBigvec(sha1("bigvec1"), HashSet() + sha1("page1"), 2, sha1("")) cppView1.addPage(sha1("page1"), (HashSet() + sha1("bigvec1")) if shouldBeInvalid else HashSet(), 1, sha1("")) cppView1.addCheckpointFile(checkpointRequest, sha1("file"), HashSet() + sha1("bigvec1"), 2, sha1("")) cppView1.addCheckpoint(checkpointRequest, HashSet() + sha1("file"), 2, sha1(""), True, 1.0, HashSet()) self.assertTrue( len(cppView1.computeInvalidObjects()) == (4 if shouldBeInvalid else 0), "%s != %s" % (len(cppView1.computeInvalidObjects()), (4 if shouldBeInvalid else 0)) ) @ComputedGraphTestHarness.UnderHarness def test_orphanedPageIsCollected(self): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler ) cppView1.addPage(sha1("page1"), HashSet(), 1, sha1("")) self.assertTrue(len(cppView1.computeInvalidObjects()) == 1) @ComputedGraphTestHarness.UnderHarness def test_basicPersistentCache(self): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler ) cppView2 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler ) cppView1.addPage(sha1("page1"), HashSet(), 1, sha1("")) cppView1.addBigvec(sha1("bigvec1"), HashSet() + sha1("page1"), 2, sha1("")) cppView1.addPage(sha1("page2"), HashSet() + sha1("bigvec1"), 3, sha1("")) cppView1.addBigvec(sha1("bigvec2"), HashSet() + sha1("page2"), 4, sha1("")) self.assertEqual(cppView1.totalBytesInCache(), 10) def seesEverything(): if not cppView2.pageExists(sha1("page1")): return False if not cppView2.pageExists(sha1("page2")): return False if not cppView2.bigvecExists(sha1("bigvec1")): return False if not cppView2.bigvecExists(sha1("bigvec2")): return False return True self.waitForSync(seesEverything) for view in [cppView1, cppView2]: self.assertEqual(view.pageBytecount(sha1("page1")), 1) self.assertEqual(view.bigvecBytecount(sha1("bigvec1")), 2) self.assertEqual(view.pageBytecount(sha1("page2")), 3) self.assertEqual(view.bigvecBytecount(sha1("bigvec2")), 4) self.assertEqual(view.totalBytesInCache(), 10) @ComputedGraphTestHarness.UnderHarness def test_writing_while_disconnected(self): currentView = [self.sharedState.newView()] cppView1 = CumulusNative.PersistentCacheIndex( currentView[0], callbackScheduler ) def writeInLoop(): for ix in range(100): time.sleep(0.01) cppView1.addPage(sha1("page" + str(ix)),HashSet(), ix, sha1("")) thread1 = threading.Thread(target=writeInLoop) thread1.start() def disconnectAndReconnectInLoop(): ix = 0 while thread1.isAlive(): ix += 1 time.sleep(0.004) currentView[0].disconnect() currentView[0] = self.sharedState.newView() cppView1.resetView(currentView[0]) thread2 = threading.Thread(target=disconnectAndReconnectInLoop) thread2.start() thread1.join() thread2.join() self.assertTrue(cppView1.timesViewReconnected() > 10) cppView2 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler ) time.sleep(2.0) count1 = 0 count2 = 0 for ix in range(100): if cppView1.pageExists(sha1("page" + str(ix))): count1 += 1 if cppView2.pageExists(sha1("page" + str(ix))): count2 += 1 self.assertTrue(count1 == 100 and count2 == 100, (count1, count2))
class TestComputedGraphSharedState(unittest.TestCase): def setUp(self): self.sharedState = SharedStateTestHarness(inMemory = True) self.keyspacePath = (0, 1) self.keyspaceName = Node.keyspacePathToKeyspaceName(self.keyspacePath) self.keyName = 'key' self.keyValue = 'value' self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", self.keyspaceName, 1) self.keyRange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False) def test_keyUpload(self): ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest(self.keyUploadTest) def test_keyDownload(self): ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest(self.keyDownloadTest) def test_propertyWrite(self): ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest(self.propertyWriteTest) def test_propertyRead(self): ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest(self.propertyReadTest) def test_synchronizerWithNoConnection(self): ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest( self.synchronizerNoConnection ) def test_synchronizerCallUpdate(self): ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest( self.synchronizerCallUpdateTest ) def test_synchronizerUpdate(self): ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest( self.synchronizerUpdateTest ) def keyUploadTest(self, computedGraphTestHarness): with self.createSynchronizer(): self.populateKey() def assertValueIsInSharedState(self, key, view): self.assertValueUpdatedInSharedState(key, view, self.keyValue) self.runInTransaction(self.keyName, assertValueIsInSharedState) def keyDownloadTest(self, computedGraphTestHarness): def updateValueInSharedState(self, key, view): view[key] = JsonPickle.toJson(self.keyValue) synchronizer = self.createSynchronizer() with synchronizer: self.runInTransaction(self.keyName, updateValueInSharedState, False) computedGraphKeyspace, computedGraphKey = self.createKey() self.synchronizerWaitForKey(self.keyName, synchronizer) self.assertEqual(computedGraphKey.value, (self.keyValue,)) def propertyWriteTest(self, computedGraphTestHarness): synchronizer = self.createSynchronizer() with synchronizer: location = TestLocation(keyspace = self.keyspacePath) location.sharedStateSubspace.keyspace.waitLoaded() location.testProperty = self.keyValue synchronizer.update() def assertValueIsInSharedState(self, key, view): self.assertValueUpdatedInSharedState(key, view, self.keyValue) self.runInTransaction( 'testProperty', assertValueIsInSharedState ) def propertyReadTest(self, computedGraphTestHarness): synchronizer = self.createSynchronizer() with synchronizer: location = TestLocation(keyspace = self.keyspacePath) def updateValueInSharedState(self, key, view): view[key] = JsonPickle.toJson(self.keyValue) self.runInTransaction( 'testProperty', updateValueInSharedState, False ) self.synchronizerWaitForKey("testProperty", synchronizer) self.assertEqual(location.testProperty, self.keyValue) def synchronizerNoConnection(self, computedGraphTestHarness): synchronizer = SharedStateSynchronizer.SharedStateSynchronizer() synchronizer.update() # This test just verifies that the call to update() doesn't blow up # when the synchronizer doesn't have a shared state view def synchronizerCallUpdateTest(self, computedGraphTestHarness): synchronizer = self.createSynchronizer() synchronizer.update() # This test just verifies that the call to update() doesn't blow up def synchronizerUpdateTest(self, computedGraphTestHarness): synchronizer = self.createSynchronizer() def updateValueInSharedState(self, key, view): view[key] = JsonPickle.toJson(self.keyValue) self.runInTransaction(self.keyName, updateValueInSharedState, False) with synchronizer: assert SharedStateSynchronizer.getView() is not None computedGraphKeyspace, computedGraphKey = self.createKey() self.synchronizerWaitForKey(self.keyName, synchronizer) self.assertEqual(computedGraphKey.value, (self.keyValue,)) def runInTransaction(self, keyPathElement, toRun, waitForKeyToLoad = True): """ Runs a specified function in a shared state transaction. toRun is a function that takes the following argumenst: self - a self reference key - the shared state key used by the test view - the a shared state view subscribed to the key range that contains key. waitForKeyToLoad - wait for SharedState to actually load the key """ view = self.sharedState.newView() view.subscribe(self.keyRange) passes = 0 while passes < 100: with SharedState.Transaction(view): key = SharedState.Key(self.keyspace, (Node.keyPathToKeyName((keyPathElement,)),)) if not waitForKeyToLoad or view[key] is not None: toRun(self, key, view) return passes += 1 time.sleep(.1) assert False, "Test timed out" def assertValueUpdatedInSharedState(self, key, view, expectedValue): self.assertIsNotNone(view[key]) value = view[key].value() self.assertEqual(JsonPickle.fromJson(value), self.keyValue) def populateKey(self): keyspace, key = self.createKey() keyspace.waitLoaded() key.value = (self.keyValue,) SharedStateSynchronizer.getSynchronizer().update() SharedStateSynchronizer.getSynchronizer().commitPendingWrites() return key def createKey(self): keyspace = CGSS.Node.Keyspace(keyspacePath = self.keyspacePath) return (keyspace, keyspace.subspace.subspace(self.keyName)) def createSynchronizer(self): synchronizer = SharedStateSynchronizer.SharedStateSynchronizer() synchronizer.attachView(self.sharedState.newView()) return synchronizer def synchronizerWaitForKey(self, keyName, synchronizer): isSet = [False] updateDict = {} keyName = Node.keyPathToKeyName((keyName,)) class Listener: def keysLoaded(listenerSelf, keyValueDict, isInitialLoad): if keyName in keyValueDict: isSet[0] = True updateDict.update(keyValueDict) synchronizer.addKeyspaceListener(self.keyspaceName, Listener(), NativeJson.Json(())) passes = 0 while not isSet[0] and passes < 100: time.sleep(.1) synchronizer.update() passes += 1 if not isSet[0]: assert False, "Failed to load %s. did see %s" % (keyName, updateDict)
class TestPersistentCacheIndex(unittest.TestCase): def setUp(self): self.sharedState = SharedStateTestHarness(inMemory=True) self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer() self.synchronizer.attachView(self.sharedState.newView()) self.synchronizer.__enter__() def tearDown(self): self.synchronizer.__exit__(None, None, None) def waitForSync(self, predicate, *synchronizers): if not synchronizers: synchronizers = [self.synchronizer] ComputedGraph.assertHasGraph() passes = 0 while passes < 100: for s in synchronizers: s.update() if predicate(): break time.sleep(.1) passes += 1 self.assertTrue(predicate()) @ComputedGraphTestHarness.UnderHarness def test_persistentCacheUnderLoad(self): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) t0 = time.time() #add 100k pages, which is enough for about 5 TB of data for index in range(100000): if index % 1000 == 0 and index > 0: print index, (time.time() - t0) / (index / 1000.0), " seconds per 1000" cppView1.addPage(sha1("page" + str(index)), HashSet(), 1, sha1("")) print "took ", time.time() - t0, " to add 100k." t1 = time.time() bytes0 = TCMallocNative.getBytesUsed() cppView2 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) while cppView2.totalBytesInCache() < 100000: time.sleep(.1) print cppView2.totalBytesInCache() print "took ", time.time() - t1, " to load 100k. Total RAM is ", (TCMallocNative.getBytesUsed() - bytes0) / 1024 / 1024.0, " MB per view"\ @ComputedGraphTestHarness.UnderHarness def test_circularPageReferencesAreInvalid(self): self.circularPageReferenceTest(True) @ComputedGraphTestHarness.UnderHarness def test_noncircularPageReferencesAreValid(self): self.circularPageReferenceTest(False) def circularPageReferenceTest(self, shouldBeInvalid): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) computationId = CumulusNative.ComputationId.Root(sha1("computation")) checkpointRequest = CumulusNative.CheckpointRequest( 0.0, True, computationId) cppView1.addBigvec(sha1("bigvec1"), HashSet() + sha1("page1"), 2, sha1("")) cppView1.addPage( sha1("page1"), (HashSet() + sha1("bigvec1")) if shouldBeInvalid else HashSet(), 1, sha1("")) cppView1.addCheckpointFile(checkpointRequest, sha1("file"), HashSet() + sha1("bigvec1"), 2, sha1("")) cppView1.addCheckpoint(checkpointRequest, HashSet() + sha1("file"), 2, sha1(""), True, 1.0, HashSet()) self.assertTrue( len(cppView1.computeInvalidObjects()) == (4 if shouldBeInvalid else 0), "%s != %s" % (len(cppView1.computeInvalidObjects()), (4 if shouldBeInvalid else 0))) @ComputedGraphTestHarness.UnderHarness def test_orphanedPageIsCollected(self): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) cppView1.addPage(sha1("page1"), HashSet(), 1, sha1("")) self.assertTrue(len(cppView1.computeInvalidObjects()) == 1) @ComputedGraphTestHarness.UnderHarness def test_basicPersistentCache(self): cppView1 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) cppView2 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) cppView1.addPage(sha1("page1"), HashSet(), 1, sha1("")) cppView1.addBigvec(sha1("bigvec1"), HashSet() + sha1("page1"), 2, sha1("")) cppView1.addPage(sha1("page2"), HashSet() + sha1("bigvec1"), 3, sha1("")) cppView1.addBigvec(sha1("bigvec2"), HashSet() + sha1("page2"), 4, sha1("")) self.assertEqual(cppView1.totalBytesInCache(), 10) def seesEverything(): if not cppView2.pageExists(sha1("page1")): return False if not cppView2.pageExists(sha1("page2")): return False if not cppView2.bigvecExists(sha1("bigvec1")): return False if not cppView2.bigvecExists(sha1("bigvec2")): return False return True self.waitForSync(seesEverything) for view in [cppView1, cppView2]: self.assertEqual(view.pageBytecount(sha1("page1")), 1) self.assertEqual(view.bigvecBytecount(sha1("bigvec1")), 2) self.assertEqual(view.pageBytecount(sha1("page2")), 3) self.assertEqual(view.bigvecBytecount(sha1("bigvec2")), 4) self.assertEqual(view.totalBytesInCache(), 10) @ComputedGraphTestHarness.UnderHarness def test_writing_while_disconnected(self): currentView = [self.sharedState.newView()] cppView1 = CumulusNative.PersistentCacheIndex(currentView[0], callbackScheduler) def writeInLoop(): for ix in range(100): time.sleep(0.01) cppView1.addPage(sha1("page" + str(ix)), HashSet(), ix, sha1("")) thread1 = threading.Thread(target=writeInLoop) thread1.start() def disconnectAndReconnectInLoop(): ix = 0 while thread1.isAlive(): ix += 1 time.sleep(0.004) currentView[0].disconnect() currentView[0] = self.sharedState.newView() cppView1.resetView(currentView[0]) thread2 = threading.Thread(target=disconnectAndReconnectInLoop) thread2.start() thread1.join() thread2.join() self.assertTrue(cppView1.timesViewReconnected() > 10) cppView2 = CumulusNative.PersistentCacheIndex( self.sharedState.newView(), callbackScheduler) time.sleep(2.0) count1 = 0 count2 = 0 for ix in range(100): if cppView1.pageExists(sha1("page" + str(ix))): count1 += 1 if cppView2.pageExists(sha1("page" + str(ix))): count2 += 1 self.assertTrue(count1 == 100 and count2 == 100, (count1, count2))