def setScriptRoots(self, scriptPath, computedValues):
hashes = Hash.ImmutableTreeSetOfHash()
for value in computedValues:
hashes = hashes + self.getComputationIdForDefinition(
value.cumulusComputationDefinition).computationHash
logging.info(
"PersistentCacheIndex setting script dependencies for %s to %s hashes",
scriptPath, len(hashes))
self.persistentCacheIndex.setScriptDependencies(scriptPath, hashes)
def test_checkpointingGarbageCollectionGraph(self):
simulation = self.createSimulation()
self.assertTrue(len(simulation.objectStore.listValues()) == 0)
try:
#give the simulation a couple of seconds to pick a scheduler
self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0))
compId1 = simulation.submitComputation(expensiveChildCachecalls(0))
compId2 = simulation.submitComputation(expensiveChildCachecalls(1))
simulation.waitForGlobalScheduler()
time.sleep(1.0)
simulation.getGlobalScheduler(
).triggerFullCheckpointsOnOutstandingComputations()
self.waitForAllCheckpointsToClear(simulation)
cache = simulation.getWorkerVdm(0).getPersistentCacheIndex()
self.assertTrue(cache.totalComputationsInCache() > 0)
self.assertEqual(cache.totalReachableComputationsInCache(), 0)
cache.setScriptDependencies(
"script",
HashNative.ImmutableTreeSetOfHash() + compId1.computationHash)
self.assertTrue(cache.totalReachableComputationsInCache() > 0)
self.assertTrue(cache.totalReachableComputationsInCache() <
cache.totalComputationsInCache())
cache.setScriptDependencies(
"script2",
HashNative.ImmutableTreeSetOfHash() + compId2.computationHash)
self.assertEqual(cache.totalComputationsInCache(),
cache.totalReachableComputationsInCache())
finally:
simulation.teardown()
def test_cumulusReconnectSharedState(self):
try:
cluster = InMemoryCluster.InMemoryCluster()
listener = WorkerCounterListener()
cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
cluster.sharedStateViewFactory
)
cumulusActiveMachines.addListener(listener)
cumulusActiveMachines.startService()
self.dialToCount(2, cluster, listener, blocking=True)
cluster.disconnectAllWorkersFromSharedState()
time.sleep(10.0)
self.dialToCount(4, cluster, listener, blocking=True)
self.assertTrue(len(cluster.cumuli), 4)
for cumulus in cluster.cumuli:
RetryAssert.waitUntilTrue(
cumulus.cumulusWorker.hasEstablishedHandshakeWithExistingMachines,
2.0)
#at this point, the persistent cache should work
persistentCacheIndex = cluster.cumuli[0].persistentCacheIndex
self.assertTrue(persistentCacheIndex.hasConnectedView())
self.assertTrue(persistentCacheIndex.timesViewReconnected() > 0)
persistentCacheIndex.addPage(HashNative.Hash.sha1("page"),
HashNative.ImmutableTreeSetOfHash(),
1,
HashNative.Hash.sha1("page"))
finally:
cumulusActiveMachines.stopService()
cluster.stop()