def test_expansionWithVecOfVec(self):
s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
4, #worker count
1,
memoryPerWorkerMB=100,
threadsPerWorker=2,
s3Service=s3)
try:
self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0))
simulation.getGlobalScheduler().setCheckpointStatusInterval(0.0001)
simulation.submitComputation(
"Vector.range(20, fun(ix) { Vector.range(100000+ix).paged }).paged"
)
simulation.waitForAnyResult()
simulation.addWorker()
self.assertTrue(simulation.waitForHandshake())
finally:
simulation.teardown()
def test_cumulusCanTriggerNewRegimes(self):
s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
4, #worker count
1,
memoryPerWorkerMB=100,
threadsPerWorker=2,
s3Service=s3)
try:
self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0))
simulation.waitForHandshake()
regime = simulation.getWorker(0).getRegimeHash()
self.assertTrue(regime is not None)
simulation.getWorker(0).triggerRegimeChange()
time.sleep(1.0)
simulation.waitForHandshake()
regime2 = simulation.getWorker(0).getRegimeHash()
self.assertTrue(regime2 is not None)
self.assertTrue(regime2 != regime)
finally:
simulation.teardown()
def loadCheckpointFromFreshSimulationTest(self,
calculationText,
timestampsPerPassList,
clientCount=1,
timestep=1.0):
s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()
statuses = []
viewFactory = None
for timestampsThisPass in timestampsPerPassList:
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
4, #worker count
clientCount,
memoryPerWorkerMB=100,
threadsPerWorker=2,
s3Service=s3,
sharedStateViewFactory=viewFactory)
viewFactory = simulation.sharedStateViewFactory
statusesThisPass = []
try:
self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0))
simulation.getGlobalScheduler().setCheckpointStatusInterval(
0.1)
for ix in range(clientCount):
simulation.submitComputationOnClient(ix, calculationText)
for subPass in range(timestampsThisPass):
time.sleep(timestep)
statusesThisPass.append(
self.timeElapsedOfMostRecentCheckpoints(simulation))
simulation.getGlobalScheduler(
).triggerFullCheckpointsOnOutstandingComputations()
self.waitForFullCheckpoint(simulation)
statusesThisPass.append(
self.timeElapsedOfMostRecentCheckpoints(simulation))
finally:
for ix in range(4):
simulation.getWorker(ix).dumpStateToLog()
simulation.teardown()
statuses.append(statusesThisPass)
return statuses
def createSimulation(self,
useHdfsObjectStore=False,
objectStore=None,
sharedStateViewFactory=None,
workerCount=4,
machineIdHashSeed=None,
s3Service=None):
s3 = s3Service or InMemoryS3Interface.InMemoryS3InterfaceFactory()
return InMemoryCumulusSimulation.InMemoryCumulusSimulation(
workerCount,
1,
memoryPerWorkerMB=100,
threadsPerWorker=2,
s3Service=s3,
objectStore=objectStore,
sharedStateViewFactory=sharedStateViewFactory,
machineIdHashSeed=machineIdHashSeed)
def DISABLEDtest_canTriggerCheckpointOfCompleted(self):
s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
4, #worker count
1,
memoryPerWorkerMB=100,
threadsPerWorker=2,
s3Service=s3)
try:
self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0))
simulation.waitForHandshake()
simulation.submitComputation("1+2")
result = simulation.waitForAnyResult()
self.assertTrue(result.isResult())
simulation.getGlobalScheduler(
).triggerFullCheckpointsOnOutstandingComputations()
ts = self.waitForFullCheckpoint(simulation, onlyUnfinished=False)
self.assertTrue(ts is not None)
statuses = simulation.getGlobalScheduler(
).currentOutstandingCheckpointStatuses(False, False)
status = statuses[0]
compId = status[0]
checkpointStatus = status[1][0]
checkpointRequest = status[1][1]
#verify that it's a storage checkpoint
self.assertTrue(checkpointRequest.writeToStorage)
self.assertTrue(checkpointStatus.checkpointSuccessful)
self.assertTrue(checkpointStatus.isRootComputationFinished)
finally:
simulation.teardown()
def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm,
**kwds):
s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory()
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
1,
0,
s3Service=s3Service,
memoryPerWorkerMB=400,
callbackScheduler=callbackScheduler,
threadsPerWorker=Setup.config().cumulusServiceThreadCount,
**kwds)
CumulusGateway.CumulusGateway.__init__(
self, callbackScheduler, vdm, simulation.sharedStateViewFactory)
self.s3Service = s3Service
self.simulation = simulation
self.viewFactory = self.simulation.sharedStateViewFactory
self.worker = self.simulation.getWorker(0)
self.workerVdm = self.simulation.getWorkerVdm(0)
channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
machineId = self.workerVdm.getMachineId()
self.cumulusClient.addMachine(machineId,
[channel1Client, channel2Client],
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
self.worker.addCumulusClient(self.cumulusClientId,
[channel1Worker, channel2Worker],
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
self.loadingService = self.simulation.loadingServices[0]
def test_dependingOnCheckpointedFinishedCachecallWorks(self):
s3 = InMemoryS3Interface.InMemoryS3InterfaceFactory()
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
4, #worker count
1,
memoryPerWorkerMB=100,
threadsPerWorker=2,
s3Service=s3)
try:
self.assertTrue(simulation.waitForGlobalScheduler(timeout=2.0))
simulation.waitForHandshake()
baseComp = simulation.createComputation("1+2")
baseCompId = simulation.getClient(0).createComputation(baseComp)
simulation.getClient(0).setComputationPriority(
baseCompId, CumulusNative.ComputationPriority(1))
result = simulation.waitForAnyResult()
self.assertTrue(result.isResult())
simulation.getGlobalScheduler(
).triggerFullCheckpointsOnOutstandingComputations()
self.waitForAllCheckpointsToClear(simulation)
for ix in range(100):
childComp = simulation.createComputation("x + %s" % ix,
x=baseComp)
childCompId = simulation.getClient(0).createComputation(
childComp)
simulation.getClient(0).setComputationPriority(
childCompId, CumulusNative.ComputationPriority(1))
result = simulation.waitForAnyResult()
finally:
simulation.teardown()
def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm,
**kwds):
#don't modify callers directly
kwds = dict(kwds)
if 's3Service' in kwds:
s3Service = kwds['s3Service']
del kwds['s3Service']
else:
s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory()
if 'threadsPerWorker' in kwds:
threadsPerWorker = kwds['threadsPerWorker']
del kwds['threadsPerWorker']
else:
threadsPerWorker = Setup.config().cumulusServiceThreadCount
if 'memoryPerWorkerMB' in kwds:
memoryPerWorkerMB = kwds['memoryPerWorkerMB']
del kwds['memoryPerWorkerMB']
else:
memoryPerWorkerMB = 400
if 'maxMBPerOutOfProcessPythonTask' in kwds:
maxBytesPerOutOfProcessPythonTask = kwds[
'maxMBPerOutOfProcessPythonTask'] * 1024 * 1024
del kwds['maxMBPerOutOfProcessPythonTask']
else:
maxBytesPerOutOfProcessPythonTask = None
workerCount = 1
if 'workerCount' in kwds:
workerCount = kwds['workerCount']
del kwds['workerCount']
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
workerCount,
0,
s3Service=s3Service,
memoryPerWorkerMB=memoryPerWorkerMB,
callbackScheduler=callbackScheduler,
threadsPerWorker=threadsPerWorker,
maxBytesPerOutOfProcessPythonTask=maxBytesPerOutOfProcessPythonTask,
**kwds)
CumulusGateway.CumulusGateway.__init__(
self, callbackScheduler, vdm, simulation.sharedStateViewFactory)
self.s3Service = s3Service
self.simulation = simulation
self.viewFactory = self.simulation.sharedStateViewFactory
for workerIx in xrange(workerCount):
worker = self.simulation.getWorker(workerIx)
workerVdm = self.simulation.getWorkerVdm(workerIx)
channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
machineId = workerVdm.getMachineId()
self.cumulusClient.addMachine(
machineId, [channel1Client, channel2Client],
ModuleImporter.builtinModuleImplVal(), self.callbackScheduler)
worker.addCumulusClient(self.cumulusClientId,
[channel1Worker, channel2Worker],
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
self.worker = self.simulation.getWorker(0)
self.workerVdm = self.simulation.getWorkerVdm(0)
self.loadingService = self.simulation.loadingServices[0]
