def dialToCount(self, cumulusCount, cluster, listener, blocking=False):
cluster.desireNumCumuli(cumulusCount, blocking=blocking)
self.assertTrue(
RetryAssert.waitUntilTrue(lambda: listener.workerCount == cumulusCount, 2.0),
"Failed to reach %s machines" % cumulusCount
)
def test_cumulusReconnectSharedState(self):
try:
cluster = InMemoryCluster.InMemoryCluster()
listener = WorkerCounterListener()
cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
cluster.sharedStateViewFactory
)
cumulusActiveMachines.addListener(listener)
cumulusActiveMachines.startService()
self.dialToCount(2, cluster, listener, blocking=True)
cluster.disconnectAllWorkersFromSharedState()
time.sleep(10.0)
self.dialToCount(4, cluster, listener, blocking=True)
self.assertTrue(len(cluster.cumuli), 4)
for cumulus in cluster.cumuli:
RetryAssert.waitUntilTrue(
cumulus.cumulusWorker.hasEstablishedHandshakeWithExistingMachines,
2.0)
#at this point, the persistent cache should work
persistentCacheIndex = cluster.cumuli[0].persistentCacheIndex
self.assertTrue(persistentCacheIndex.hasConnectedView())
self.assertTrue(persistentCacheIndex.timesViewReconnected() > 0)
persistentCacheIndex.addPage(HashNative.Hash.sha1("page"),
HashNative.ImmutableTreeSetOfHash(),
1,
HashNative.Hash.sha1("page"))
finally:
cumulusActiveMachines.stopService()
cluster.stop()
def assertExpected(self, views):
getTable = lambda : self.getIntrospectionTable(views)
expected = lambda : self.getExpectedFromViewList(views)
RetryAssert.retryAssert(self.assertDictsSame, [getTable, expected], numRetries=50)
