def setUp(self):
self.sharedState = SharedStateTestHarness(inMemory = True)
self.keyspacePath = (0, 1)
self.keyspaceName = Node.keyspacePathToKeyspaceName(self.keyspacePath)
self.keyName = 'key'
self.keyValue = 'value'
self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", self.keyspaceName, 1)
self.keyRange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False)
def setUp(self):
self.sharedState = SharedStateTestHarness(inMemory=True)
self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
self.synchronizer.attachView(self.sharedState.newView())
self.synchronizer.__enter__()
class TestPersistentCacheIndex(unittest.TestCase):
def setUp(self):
self.sharedState = SharedStateTestHarness(inMemory=True)
self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
self.synchronizer.attachView(self.sharedState.newView())
self.synchronizer.__enter__()
def tearDown(self):
self.synchronizer.__exit__(None, None, None)
def waitForSync(self, predicate, *synchronizers):
if not synchronizers:
synchronizers = [self.synchronizer]
ComputedGraph.assertHasGraph()
passes = 0
while passes < 100:
for s in synchronizers:
s.update()
if predicate():
break
time.sleep(.1)
passes += 1
self.assertTrue(predicate())
@ComputedGraphTestHarness.UnderHarness
def test_persistentCacheUnderLoad(self):
cppView1 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(),
callbackScheduler
)
t0 = time.time()
#add 100k pages, which is enough for about 5 TB of data
for index in range(100000):
if index % 1000 == 0 and index > 0:
print index, (time.time() - t0) / (index / 1000.0), " seconds per 1000"
cppView1.addPage(sha1("page" + str(index)), HashSet(), 1, sha1(""))
print "took ", time.time() - t0, " to add 100k."
t1 = time.time()
bytes0 = TCMallocNative.getBytesUsed()
cppView2 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(),
callbackScheduler
)
while cppView2.totalBytesInCache() < 100000:
time.sleep(.1)
print cppView2.totalBytesInCache()
print "took ", time.time() - t1, " to load 100k. Total RAM is ", (TCMallocNative.getBytesUsed() - bytes0) / 1024 / 1024.0, " MB per view"\
@ComputedGraphTestHarness.UnderHarness
def test_circularPageReferencesAreInvalid(self):
self.circularPageReferenceTest(True)
@ComputedGraphTestHarness.UnderHarness
def test_noncircularPageReferencesAreValid(self):
self.circularPageReferenceTest(False)
def circularPageReferenceTest(self, shouldBeInvalid):
cppView1 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(),
callbackScheduler
)
computationId = CumulusNative.ComputationId.Root(
sha1("computation")
)
checkpointRequest = CumulusNative.CheckpointRequest(0.0, True, computationId)
cppView1.addBigvec(sha1("bigvec1"), HashSet() + sha1("page1"), 2, sha1(""))
cppView1.addPage(sha1("page1"), (HashSet() + sha1("bigvec1")) if shouldBeInvalid else HashSet(), 1, sha1(""))
cppView1.addCheckpointFile(checkpointRequest, sha1("file"), HashSet() + sha1("bigvec1"), 2, sha1(""))
cppView1.addCheckpoint(checkpointRequest, HashSet() + sha1("file"), 2, sha1(""), True, 1.0, HashSet())
self.assertTrue(
len(cppView1.computeInvalidObjects()) ==
(4 if shouldBeInvalid else 0),
"%s != %s" % (len(cppView1.computeInvalidObjects()), (4 if shouldBeInvalid else 0))
)
@ComputedGraphTestHarness.UnderHarness
def test_orphanedPageIsCollected(self):
cppView1 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(),
callbackScheduler
)
cppView1.addPage(sha1("page1"), HashSet(), 1, sha1(""))
self.assertTrue(len(cppView1.computeInvalidObjects()) == 1)
@ComputedGraphTestHarness.UnderHarness
def test_basicPersistentCache(self):
cppView1 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(),
callbackScheduler
)
cppView2 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(),
callbackScheduler
)
cppView1.addPage(sha1("page1"), HashSet(), 1, sha1(""))
cppView1.addBigvec(sha1("bigvec1"), HashSet() + sha1("page1"), 2, sha1(""))
cppView1.addPage(sha1("page2"), HashSet() + sha1("bigvec1"), 3, sha1(""))
cppView1.addBigvec(sha1("bigvec2"), HashSet() + sha1("page2"), 4, sha1(""))
self.assertEqual(cppView1.totalBytesInCache(), 10)
def seesEverything():
if not cppView2.pageExists(sha1("page1")):
return False
if not cppView2.pageExists(sha1("page2")):
return False
if not cppView2.bigvecExists(sha1("bigvec1")):
return False
if not cppView2.bigvecExists(sha1("bigvec2")):
return False
return True
self.waitForSync(seesEverything)
for view in [cppView1, cppView2]:
self.assertEqual(view.pageBytecount(sha1("page1")), 1)
self.assertEqual(view.bigvecBytecount(sha1("bigvec1")), 2)
self.assertEqual(view.pageBytecount(sha1("page2")), 3)
self.assertEqual(view.bigvecBytecount(sha1("bigvec2")), 4)
self.assertEqual(view.totalBytesInCache(), 10)
@ComputedGraphTestHarness.UnderHarness
def test_writing_while_disconnected(self):
currentView = [self.sharedState.newView()]
cppView1 = CumulusNative.PersistentCacheIndex(
currentView[0],
callbackScheduler
)
def writeInLoop():
for ix in range(100):
time.sleep(0.01)
cppView1.addPage(sha1("page" + str(ix)),HashSet(), ix, sha1(""))
thread1 = threading.Thread(target=writeInLoop)
thread1.start()
def disconnectAndReconnectInLoop():
ix = 0
while thread1.isAlive():
ix += 1
time.sleep(0.004)
currentView[0].disconnect()
currentView[0] = self.sharedState.newView()
cppView1.resetView(currentView[0])
thread2 = threading.Thread(target=disconnectAndReconnectInLoop)
thread2.start()
thread1.join()
thread2.join()
self.assertTrue(cppView1.timesViewReconnected() > 10)
cppView2 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(),
callbackScheduler
)
time.sleep(2.0)
count1 = 0
count2 = 0
for ix in range(100):
if cppView1.pageExists(sha1("page" + str(ix))):
count1 += 1
if cppView2.pageExists(sha1("page" + str(ix))):
count2 += 1
self.assertTrue(count1 == 100 and count2 == 100, (count1, count2))
class TestComputedGraphSharedState(unittest.TestCase):
def setUp(self):
self.sharedState = SharedStateTestHarness(inMemory = True)
self.keyspacePath = (0, 1)
self.keyspaceName = Node.keyspacePathToKeyspaceName(self.keyspacePath)
self.keyName = 'key'
self.keyValue = 'value'
self.keyspace = SharedState.Keyspace("TakeHighestIdKeyType", self.keyspaceName, 1)
self.keyRange = SharedState.KeyRange(self.keyspace, 0, None, None, True, False)
def test_keyUpload(self):
ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest(self.keyUploadTest)
def test_keyDownload(self):
ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest(self.keyDownloadTest)
def test_propertyWrite(self):
ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest(self.propertyWriteTest)
def test_propertyRead(self):
ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest(self.propertyReadTest)
def test_synchronizerWithNoConnection(self):
ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest(
self.synchronizerNoConnection
)
def test_synchronizerCallUpdate(self):
ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest(
self.synchronizerCallUpdateTest
)
def test_synchronizerUpdate(self):
ComputedGraphTestHarness.ComputedGraphTestHarness().executeTest(
self.synchronizerUpdateTest
)
def keyUploadTest(self, computedGraphTestHarness):
with self.createSynchronizer():
self.populateKey()
def assertValueIsInSharedState(self, key, view):
self.assertValueUpdatedInSharedState(key, view, self.keyValue)
self.runInTransaction(self.keyName, assertValueIsInSharedState)
def keyDownloadTest(self, computedGraphTestHarness):
def updateValueInSharedState(self, key, view):
view[key] = JsonPickle.toJson(self.keyValue)
synchronizer = self.createSynchronizer()
with synchronizer:
self.runInTransaction(self.keyName, updateValueInSharedState, False)
computedGraphKeyspace, computedGraphKey = self.createKey()
self.synchronizerWaitForKey(self.keyName, synchronizer)
self.assertEqual(computedGraphKey.value, (self.keyValue,))
def propertyWriteTest(self, computedGraphTestHarness):
synchronizer = self.createSynchronizer()
with synchronizer:
location = TestLocation(keyspace = self.keyspacePath)
location.sharedStateSubspace.keyspace.waitLoaded()
location.testProperty = self.keyValue
synchronizer.update()
def assertValueIsInSharedState(self, key, view):
self.assertValueUpdatedInSharedState(key, view, self.keyValue)
self.runInTransaction(
'testProperty',
assertValueIsInSharedState
)
def propertyReadTest(self, computedGraphTestHarness):
synchronizer = self.createSynchronizer()
with synchronizer:
location = TestLocation(keyspace = self.keyspacePath)
def updateValueInSharedState(self, key, view):
view[key] = JsonPickle.toJson(self.keyValue)
self.runInTransaction(
'testProperty',
updateValueInSharedState,
False
)
self.synchronizerWaitForKey("testProperty", synchronizer)
self.assertEqual(location.testProperty, self.keyValue)
def synchronizerNoConnection(self, computedGraphTestHarness):
synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
synchronizer.update()
# This test just verifies that the call to update() doesn't blow up
# when the synchronizer doesn't have a shared state view
def synchronizerCallUpdateTest(self, computedGraphTestHarness):
synchronizer = self.createSynchronizer()
synchronizer.update()
# This test just verifies that the call to update() doesn't blow up
def synchronizerUpdateTest(self, computedGraphTestHarness):
synchronizer = self.createSynchronizer()
def updateValueInSharedState(self, key, view):
view[key] = JsonPickle.toJson(self.keyValue)
self.runInTransaction(self.keyName, updateValueInSharedState, False)
with synchronizer:
assert SharedStateSynchronizer.getView() is not None
computedGraphKeyspace, computedGraphKey = self.createKey()
self.synchronizerWaitForKey(self.keyName, synchronizer)
self.assertEqual(computedGraphKey.value, (self.keyValue,))
def runInTransaction(self, keyPathElement, toRun, waitForKeyToLoad = True):
"""
Runs a specified function in a shared state transaction.
toRun is a function that takes the following argumenst:
self - a self reference
key - the shared state key used by the test
view - the a shared state view subscribed to the key range
that contains key.
waitForKeyToLoad - wait for SharedState to actually load the
key
"""
view = self.sharedState.newView()
view.subscribe(self.keyRange)
passes = 0
while passes < 100:
with SharedState.Transaction(view):
key = SharedState.Key(self.keyspace, (Node.keyPathToKeyName((keyPathElement,)),))
if not waitForKeyToLoad or view[key] is not None:
toRun(self, key, view)
return
passes += 1
time.sleep(.1)
assert False, "Test timed out"
def assertValueUpdatedInSharedState(self, key, view, expectedValue):
self.assertIsNotNone(view[key])
value = view[key].value()
self.assertEqual(JsonPickle.fromJson(value), self.keyValue)
def populateKey(self):
keyspace, key = self.createKey()
keyspace.waitLoaded()
key.value = (self.keyValue,)
SharedStateSynchronizer.getSynchronizer().update()
SharedStateSynchronizer.getSynchronizer().commitPendingWrites()
return key
def createKey(self):
keyspace = CGSS.Node.Keyspace(keyspacePath = self.keyspacePath)
return (keyspace, keyspace.subspace.subspace(self.keyName))
def createSynchronizer(self):
synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
synchronizer.attachView(self.sharedState.newView())
return synchronizer
def synchronizerWaitForKey(self, keyName, synchronizer):
isSet = [False]
updateDict = {}
keyName = Node.keyPathToKeyName((keyName,))
class Listener:
def keysLoaded(listenerSelf, keyValueDict, isInitialLoad):
if keyName in keyValueDict:
isSet[0] = True
updateDict.update(keyValueDict)
synchronizer.addKeyspaceListener(self.keyspaceName, Listener(), NativeJson.Json(()))
passes = 0
while not isSet[0] and passes < 100:
time.sleep(.1)
synchronizer.update()
passes += 1
if not isSet[0]:
assert False, "Failed to load %s. did see %s" % (keyName, updateDict)
def setUp(self):
self.sharedState = SharedStateTestHarness(inMemory=True)
self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
self.synchronizer.attachView(self.sharedState.newView())
self.synchronizer.__enter__()
class TestPersistentCacheIndex(unittest.TestCase):
def setUp(self):
self.sharedState = SharedStateTestHarness(inMemory=True)
self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
self.synchronizer.attachView(self.sharedState.newView())
self.synchronizer.__enter__()
def tearDown(self):
self.synchronizer.__exit__(None, None, None)
def waitForSync(self, predicate, *synchronizers):
if not synchronizers:
synchronizers = [self.synchronizer]
ComputedGraph.assertHasGraph()
passes = 0
while passes < 100:
for s in synchronizers:
s.update()
if predicate():
break
time.sleep(.1)
passes += 1
self.assertTrue(predicate())
@ComputedGraphTestHarness.UnderHarness
def test_persistentCacheUnderLoad(self):
cppView1 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(), callbackScheduler)
t0 = time.time()
#add 100k pages, which is enough for about 5 TB of data
for index in range(100000):
if index % 1000 == 0 and index > 0:
print index, (time.time() - t0) / (index /
1000.0), " seconds per 1000"
cppView1.addPage(sha1("page" + str(index)), HashSet(), 1, sha1(""))
print "took ", time.time() - t0, " to add 100k."
t1 = time.time()
bytes0 = TCMallocNative.getBytesUsed()
cppView2 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(), callbackScheduler)
while cppView2.totalBytesInCache() < 100000:
time.sleep(.1)
print cppView2.totalBytesInCache()
print "took ", time.time() - t1, " to load 100k. Total RAM is ", (TCMallocNative.getBytesUsed() - bytes0) / 1024 / 1024.0, " MB per view"\
@ComputedGraphTestHarness.UnderHarness
def test_circularPageReferencesAreInvalid(self):
self.circularPageReferenceTest(True)
@ComputedGraphTestHarness.UnderHarness
def test_noncircularPageReferencesAreValid(self):
self.circularPageReferenceTest(False)
def circularPageReferenceTest(self, shouldBeInvalid):
cppView1 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(), callbackScheduler)
computationId = CumulusNative.ComputationId.Root(sha1("computation"))
checkpointRequest = CumulusNative.CheckpointRequest(
0.0, True, computationId)
cppView1.addBigvec(sha1("bigvec1"),
HashSet() + sha1("page1"), 2, sha1(""))
cppView1.addPage(
sha1("page1"),
(HashSet() + sha1("bigvec1")) if shouldBeInvalid else HashSet(), 1,
sha1(""))
cppView1.addCheckpointFile(checkpointRequest, sha1("file"),
HashSet() + sha1("bigvec1"), 2, sha1(""))
cppView1.addCheckpoint(checkpointRequest,
HashSet() + sha1("file"), 2, sha1(""), True,
1.0, HashSet())
self.assertTrue(
len(cppView1.computeInvalidObjects()) == (4 if shouldBeInvalid else
0),
"%s != %s" % (len(cppView1.computeInvalidObjects()),
(4 if shouldBeInvalid else 0)))
@ComputedGraphTestHarness.UnderHarness
def test_orphanedPageIsCollected(self):
cppView1 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(), callbackScheduler)
cppView1.addPage(sha1("page1"), HashSet(), 1, sha1(""))
self.assertTrue(len(cppView1.computeInvalidObjects()) == 1)
@ComputedGraphTestHarness.UnderHarness
def test_basicPersistentCache(self):
cppView1 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(), callbackScheduler)
cppView2 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(), callbackScheduler)
cppView1.addPage(sha1("page1"), HashSet(), 1, sha1(""))
cppView1.addBigvec(sha1("bigvec1"),
HashSet() + sha1("page1"), 2, sha1(""))
cppView1.addPage(sha1("page2"),
HashSet() + sha1("bigvec1"), 3, sha1(""))
cppView1.addBigvec(sha1("bigvec2"),
HashSet() + sha1("page2"), 4, sha1(""))
self.assertEqual(cppView1.totalBytesInCache(), 10)
def seesEverything():
if not cppView2.pageExists(sha1("page1")):
return False
if not cppView2.pageExists(sha1("page2")):
return False
if not cppView2.bigvecExists(sha1("bigvec1")):
return False
if not cppView2.bigvecExists(sha1("bigvec2")):
return False
return True
self.waitForSync(seesEverything)
for view in [cppView1, cppView2]:
self.assertEqual(view.pageBytecount(sha1("page1")), 1)
self.assertEqual(view.bigvecBytecount(sha1("bigvec1")), 2)
self.assertEqual(view.pageBytecount(sha1("page2")), 3)
self.assertEqual(view.bigvecBytecount(sha1("bigvec2")), 4)
self.assertEqual(view.totalBytesInCache(), 10)
@ComputedGraphTestHarness.UnderHarness
def test_writing_while_disconnected(self):
currentView = [self.sharedState.newView()]
cppView1 = CumulusNative.PersistentCacheIndex(currentView[0],
callbackScheduler)
def writeInLoop():
for ix in range(100):
time.sleep(0.01)
cppView1.addPage(sha1("page" + str(ix)), HashSet(), ix,
sha1(""))
thread1 = threading.Thread(target=writeInLoop)
thread1.start()
def disconnectAndReconnectInLoop():
ix = 0
while thread1.isAlive():
ix += 1
time.sleep(0.004)
currentView[0].disconnect()
currentView[0] = self.sharedState.newView()
cppView1.resetView(currentView[0])
thread2 = threading.Thread(target=disconnectAndReconnectInLoop)
thread2.start()
thread1.join()
thread2.join()
self.assertTrue(cppView1.timesViewReconnected() > 10)
cppView2 = CumulusNative.PersistentCacheIndex(
self.sharedState.newView(), callbackScheduler)
time.sleep(2.0)
count1 = 0
count2 = 0
for ix in range(100):
if cppView1.pageExists(sha1("page" + str(ix))):
count1 += 1
if cppView2.pageExists(sha1("page" + str(ix))):
count2 += 1
self.assertTrue(count1 == 100 and count2 == 100, (count1, count2))
