def test_verifyThatExtractingPausedComputationsDoesntDuplicateLargeStrings(
self):
text = """fun() {
let s = ' '
while (size(s) < 1000000)
s = s + s
let f = fun(x) { if (x > 0) return f(x-1) + s[x]; `TriggerInterruptForTesting() }
f(20)
}"""
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
computation = context.extractPausedComputation()
context2 = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context2.resumePausedComputation(computation)
self.assertTrue(context2.totalBytesUsed < 2 * context.totalBytesUsed)
def test_resumingAfterCopyDataOutOfPages(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager=vdm)
context.interruptAfterCycleCount(100000)
text = """
fun() {
let v = Vector.range(1000).paged;
let ix1 = 0
let res = 0
while (true) {
res = res + v[ix1]
ix1 = (ix1 + 1) % size(v)
}
res
}"""
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
paused1 = context.extractPausedComputation()
while not context.isVectorLoad():
context.copyValuesOutOfVectorPages()
vdm.unloadAllPossible()
context.interruptAfterCycleCount(100000)
context.resume()
paused2 = context.extractPausedComputation()
self.assertTrue(len(paused1.frames) == len(paused2.frames))
def _test_vectorElementJOV_VectorElementJOVFromLiveValue_compatibility(self):
vdmSimple = None # Required to pass lint check - represents a bug
jovs = self.sampleJOVs()
numpy.random.seed(42)
context = ExecutionContext.ExecutionContext(
32 * 1024,
False,
False,
vdmSimple
)
randomJOVGenerator = FORANative.RandomJOVGenerator(0,
context).symbolStrings(self.symbol_strings)
for jov in jovs:
for i in range(100):
random_value = randomJOVGenerator.randomValue(jov)
if random_value != None:
vectorElementJOVFromLiveValue = \
FORANative.VectorElementJOVFromLiveValue(random_value)
vectorElementJOVFromJOV = \
FORANative.JOVFromLiveValue(random_value).vectorElementJOV()
if vectorElementJOVFromLiveValue != vectorElementJOVFromJOV:
print "rand_val = %s" \
% random_value
print "vectorElementJOVFromLiveValue = %s" \
% vectorElementJOVFromLiveValue
print "vectorElementJOVFromJOV = %s" \
% vectorElementJOVFromJOV
print "JOVFromLiveValue = %s" \
% FORANative.JOVFromLiveValue(random_value)
self.assertEqual(vectorElementJOVFromLiveValue, vectorElementJOVFromJOV)
def test_teardown_simple(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInternalSplitting=False)
evaluate(context,
FORA.extractImplValContainer(FORA.eval("fun(){nothing}")),
FORANative.symbol_Call)
context.getFinishedResult()
toEval = FORA.extractImplValContainer(
FORA.eval("""fun() {
let f = fun() { };
let v = [1, [3]];
cached(f())
}"""))
evaluate(context, toEval, FORANative.symbol_Call)
while not context.isCacheRequest():
context.compute()
context.teardown(True)
def test_extractPausedComputationDuringVectorLoad(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call")))
pagedVec = context.getFinishedResult().asResult.result
context.placeInEvaluationState(
FORANative.ImplValContainer(
(pagedVec,
FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
FORANative.ImplValContainer(0))))
vdm.unloadAllPossible()
context.resume()
self.assertTrue(context.isVectorLoad())
computation = context.extractPausedComputation()
self.assertEqual(len(computation.frames), 1)
def createWorker(machineId,
viewFactory,
callbackSchedulerToUse=None,
threadCount=2,
memoryLimitMb=100):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024)
vdm.setMemoryLimit(
int(memoryLimitMb * 1024 * 1024),
min(int(memoryLimitMb * 1.25 * 1024 * 1024),
int((memoryLimitMb + 1024 * 2) * 1024 * 1024)))
vdm.setPersistentCacheIndex(
CumulusNative.PersistentCacheIndex(viewFactory.createView(),
callbackSchedulerToUse))
cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse,
1000 * 1024 * 1024)
eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler()
return (CumulusNative.CumulusWorker(
callbackSchedulerToUse,
CumulusNative.CumulusWorkerConfiguration(
machineId, threadCount,
CumulusNative.CumulusCheckpointPolicy.None (),
ExecutionContext.createContextConfiguration(), ""), vdm, cache,
eventHandler), vdm, eventHandler)
def test_serialize_while_holding_interior_vector(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm,
allowInterpreterTracing=False,
allowInternalSplitting=False)
evaluate(
context,
FORA.extractImplValContainer(
FORA.eval("""
fun() {
let v = [[1].paged].paged;
let v2 = v[0]
`TriggerInterruptForTesting()
1+2+3+v+v2
}""")), FORANative.symbol_Call)
self.assertTrue(context.isInterrupted())
serialized = context.serialize()
context = None
def test_refcountsInCompiledCode(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm,
allowInterpreterTracing=True,
blockUntilTracesAreCompiled=True)
text = """fun(){
let f = fun(v, depth) {
if (depth > 100)
//this will trigger an interrupt since the data cannot exist in the VDM
datasets.s3('','')
else
f(v, depth+1)
}
f([1,2,3,4,5], 0)
}"""
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
stacktraceText = context.extractCurrentTextStacktrace()
self.assertTrue(stacktraceText.count("Vector") < 10)
def copyDataOutOfPagesTest(self, text, cycleCount, expectsToHaveCopies):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm,
allowInterpreterTracing=False,
allowInternalSplitting=False)
context.configuration.agressivelyValidateRefcountsAndPageReachability = True
context.configuration.releaseVectorHandlesImmediatelyAfterExecution = False
context.placeInEvaluationState(
FORANative.ImplValContainer(
(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)))
context.interruptAfterCycleCount(cycleCount)
context.compute()
if expectsToHaveCopies:
self.assertTrue(context.copyValuesOutOfVectorPages())
self.assertFalse(context.copyValuesOutOfVectorPages())
else:
self.assertFalse(context.copyValuesOutOfVectorPages())
def test_resumePausedComputationWithResult(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm,
allowInterpreterTracing=False,
allowInternalSplitting=False)
text = """
let f = fun(v, ix) {
if (ix > 0)
{
let (v2,res) = f(v,ix-1);
return (v2, res + v2[0])
}
`TriggerInterruptForTesting()
return (v, 0)
};
f([1], 10)
"""
evaluate(
context,
FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call")))
assert context.isInterrupted()
pausedComp = context.extractPausedComputation()
framesToUse = pausedComp.asThread.computation.frames[0:5]
pausedComp2 = FORANative.PausedComputationTree(
FORANative.PausedComputation(
framesToUse,
FORA.extractImplValContainer(
FORA.eval("([2], 0)", keepAsForaValue=True)), False))
context.resumePausedComputation(pausedComp2)
context.copyValuesOutOfVectorPages()
context.pageLargeVectorHandles(0)
context.resetInterruptState()
context.compute()
self.assertTrue(context.isFinished())
result = context.getFinishedResult()
self.assertTrue(result.asResult.result[1].pyval == 6)
def grabContext(self):
"""get an execution context"""
with self.lock_:
if not self.contexts_:
context = ExecutionContext.ExecutionContext(
dataManager=self.vdm_)
else:
context = self.contexts_.pop()
return context
def test_serialize_during_vector_load(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager=vdm)
context.evaluate(
FORA.extractImplValContainer(
FORA.eval("fun(){ datasets.s3('a','b')[0] }")),
FORANative.symbol_Call)
self.assertTrue(context.isVectorLoad())
serialized = context.serialize()
context2 = ExecutionContext.ExecutionContext(dataManager=vdm)
context2.deserialize(serialized)
self.assertTrue(context2.isVectorLoad())
def createWorker_(machineId,
viewFactory,
callbackSchedulerToUse,
threadCount,
memoryLimitMb,
cacheFunction,
pageSizeOverride,
disableEventHandler):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
vdm = ForaNative.VectorDataManager(
callbackSchedulerToUse,
pageSizeOverride if pageSizeOverride is not None else
1 * 1024 * 1024 if memoryLimitMb < 1000 else
5 * 1024 * 1024 if memoryLimitMb < 5000 else
50 * 1024 * 1024
)
vdm.setMemoryLimit(
int(memoryLimitMb * 1024 * 1024),
min(int(memoryLimitMb * 1.25 * 1024 * 1024),
int((memoryLimitMb + 1024 * 2) * 1024 * 1024))
)
vdm.setPersistentCacheIndex(
CumulusNative.PersistentCacheIndex(
viewFactory.createView(),
callbackSchedulerToUse
)
)
cache = cacheFunction()
if disableEventHandler:
eventHandler = CumulusNative.CumulusWorkerIgnoreEventHandler()
else:
eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler()
return (
CumulusNative.CumulusWorker(
callbackSchedulerToUse,
CumulusNative.CumulusWorkerConfiguration(
machineId,
threadCount,
CumulusNative.CumulusCheckpointPolicy.None(),
ExecutionContext.createContextConfiguration(),
""
),
vdm,
cache,
eventHandler
),
vdm,
eventHandler
)
def test_covers(self):
vdm = FORANative.VectorDataManager(
callbackScheduler,
Setup.config().maxPageSizeInBytes
)
context = ExecutionContext.ExecutionContext(dataManager = vdm)
randomJOVGenerator = FORANative.RandomJOVGenerator(0, context)
self.treeElementCoveringTest(coverageTree, [], [], randomJOVGenerator)
def callAndGetResult(funImplVal):
vdm = ForaNative.VectorDataManager(callbackScheduler, 50 * 1024 * 1024)
context = ExecutionContext.ExecutionContext(dataManager=vdm,
allowInterpreterTracing=False)
context.evaluate(funImplVal, ForaNative.symbol_Call)
finishedResult = context.getFinishedResult()
return finishedResult
def runSimpleEvaluation(self, inputType, outputType):
mainRuntime = Runtime.getMainRuntime()
foraCompiler = mainRuntime.getTypedForaCompiler()
while foraCompiler.anyCompilingOrPending():
time.sleep(.01)
aParticularStringValue = ForaNative.ImplValContainer(aBigString)
callable = self.generateSimpleCallable(inputType, outputType)
jumpTarget = foraCompiler.compile(callable)
import gc
gc.collect()
for passIndex in range(PASS_COUNT):
#type values are memoized, so we can't assume that the value has a refcount
# of exactly one
totalStringCount = ForaNative.totalStringCount()
totalImplvalCount = ForaNative.totalImplvalCount()
anExecutionContext = ExecutionContext.ExecutionContext(
dataManager=VectorDataManager.constructVDM(
self.callbackScheduler))
anExecutionContext.evaluateFunctionPointer(jumpTarget,
aParticularStringValue)
self.assertTrue(anExecutionContext.isFinished())
res = anExecutionContext.getFinishedResult()
self.assertTrue(not res.isException())
self.assertEqual(res.asResult.result, aParticularStringValue)
anExecutionContext.teardown()
res = None
#verify final refcounts
self.assertEqual(
aParticularStringValue.getStringObjectRefcount(), 1,
"refcounts weren't maintained in %s->%s. %s != 1" %
(inputType, outputType,
aParticularStringValue.getStringObjectRefcount()))
self.assertEqual((totalStringCount, totalImplvalCount),
(ForaNative.totalStringCount(),
ForaNative.totalImplvalCount()),
"refcounts weren't maintained in " +
str(inputType) + "->" + str(outputType))
def test_extractPausedComputation(self):
text = """fun() {
let x = 0;
while (x < 100000)
x = x + 1
x
}"""
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.interruptAfterCycleCount(1010)
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
computation = context.extractPausedComputation()
self.assertEqual(len(computation.frames), 2)
self.assertEqual(computation.frames[1].values[0].pyval, 335)
context2 = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context2.resumePausedComputation(computation)
context2.resume()
self.assertEqual(context2.getFinishedResult().asResult.result.pyval,
100000)
context.teardown()
context2.teardown()
def test_teardown_simple_2(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInternalSplitting=False)
context.placeInEvaluationState(
FORANative.ImplValContainer((FORA.extractImplValContainer(
FORA.eval(
"fun(){ let f = fun() { throw 1 }; try { f() } catch(...) { throw 2 } }"
)), FORANative.symbol_Call)))
context.compute()
self.assertTrue(context.getFinishedResult().isException())
def finishPausedComputation(pausedComputation):
vdm = ForaNative.VectorDataManager(callbackScheduler, 50 * 1024 * 1024)
context2 = ExecutionContext.ExecutionContext(dataManager=vdm,
allowInterpreterTracing=False)
context2.resumePausedComputation(pausedComputation)
context2.resume()
if (not context2.isFinished()):
raise CouldntFinishException(pausedComputation)
finishedResult = context2.getFinishedResult()
return finishedResult
def extractPagedUnloadedVector(self, vdm, count):
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInternalSplitting=False)
evaluate(
context,
FORA.extractImplValContainer(
FORA.eval("fun() { Vector.range(%s).paged }" % count)),
FORANative.ImplValContainer(FORANative.makeSymbol("Call")))
pagedVec = context.getFinishedResult().asResult.result
vdm.unloadAllPossible()
return pagedVec
def callAndExtractPausedCompuationAfterSteps(funToCall, steps):
vdm = ForaNative.VectorDataManager(callbackScheduler, 50 * 1024 * 1024)
context = ExecutionContext.ExecutionContext(dataManager=vdm,
allowInterpreterTracing=False)
context.interruptAfterCycleCount(steps)
context.evaluate(funToCall, ForaNative.symbol_Call)
if (not context.isInterrupted()):
raise NotInterruptedException(context)
computation = context.extractPausedComputation()
context.teardown()
return computation
def test_extractStacktrace(self):
text = """fun() {
let x = 0;
let v = [1,2,3]
while (x < 100000) {
x = x + 1
}
(x,v)
}"""
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.interruptAfterCycleCount(1000)
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
trace = context.extractStacktrace(True)
self.assertTrue(len(trace) == 1)
stacktrace = trace[0][0]
frameData = trace[0][1]
codeLocation = FORANative.getCodeLocation(stacktrace.getIDs()[0])
self.assertTrue(codeLocation is not None)
self.assertTrue(frameData.wasCompiled is False)
self.assertTrue(len(frameData.inScopeValues) == 2)
self.assertTrue([str(x)
for x in frameData.inScopeValues.keys] == ['x', 'v'])
values = [
frameData.inScopeValues[x] for x in frameData.inScopeValues.keys
]
self.assertTrue(values[0].isInt64())
self.assertTrue(values[1].isVector())
context.teardown()
def test_interrupt_works(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager=vdm)
triggerAfter(context.interrupt, .03)
t0 = time.time()
context.evaluate(self.loopFun, FORANative.symbol_Call)
#make sure we actually looped!
self.assertTrue(time.time() - t0 > .02)
self.assertFalse(context.isEmpty())
self.assertFalse(context.isCacheRequest())
self.assertFalse(context.isVectorLoad())
self.assertFalse(context.isFinished())
self.assertTrue(context.isInterrupted())
def runSimpleEvaluation(self, callable, arguments, validator):
mainRuntime = Runtime.getMainRuntime()
foraCompiler = mainRuntime.getTypedForaCompiler()
jumpTarget = foraCompiler.compile(callable)
while foraCompiler.anyCompilingOrPending():
time.sleep(.01)
gc.collect()
for passIndex in range(PASS_COUNT):
totalStringCount = ForaNative.totalStringCount()
totalImplvalCount = ForaNative.totalImplvalCount()
anExecutionContext = ExecutionContext.ExecutionContext(
dataManager = VectorDataManager.constructVDM(self.callbackScheduler)
)
anExecutionContext.evaluateFunctionPointer(jumpTarget, *arguments)
self.assertTrue(anExecutionContext.isFinished())
res = anExecutionContext.getFinishedResult()
self.assertTrue(validator(res.asResult.result),
"invalid result in " + str(callable) + " with " + str(arguments) +
". got " + str(res)
)
res = None
anExecutionContext.teardown()
curRefs = (ForaNative.totalStringCount(), ForaNative.totalImplvalCount())
self.assertEqual(
(totalStringCount, totalImplvalCount),
curRefs,
"refcounts weren't maintained in " + str(callable) + " with " + str(arguments) +
". %s != %s" % (
(totalStringCount, totalImplvalCount),
curRefs
)
)
def pageLargeVectorHandlesTest(self, text, cycleCount, expectsToHavePages):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.configuration.agressivelyValidateRefcountsAndPageReachability = True
context.interruptAfterCycleCount(cycleCount)
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
if expectsToHavePages:
self.assertTrue(context.pageLargeVectorHandles(0))
self.assertFalse(context.pageLargeVectorHandles(0))
else:
self.assertFalse(context.pageLargeVectorHandles(0))
return context
def test_axiom_consistency(self):
vdm = FORANative.VectorDataManager(
self.callbackScheduler,
Setup.config().maxPageSizeInBytes
)
self.context = ExecutionContext.ExecutionContext(dataManager = vdm)
self.randomJOVGenerator = FORANative.RandomJOVGenerator(
self.seed, self.context).symbolStrings(self.symbol_strings).setMaxStringLength(50)
for jovt in self.axiom_signatures_to_test:
axiom = self.axioms.getAxiomByJOVT(self.typed_fora_compiler, jovt)
self.assertIsNotNone(axiom, "unable to find an axiom for the signature %s" %jovt)
if axiom.isExpansion():
assert False, "We can only test native expansions."
elif axiom.isNativeCall():
self.check_native_axiom_consistency(axiom)
else:
assert False, "Unexpected axiom type in signature %s" % jovt
def _test_jov_random_value(self):
vdmSimple = None # Required to pass lint check - represents a bug
context = ExecutionContext.ExecutionContext(
32 * 1024,
False,
False,
vdmSimple
)
jovs = self.sampleJOVs()
randomJOVGenerator = FORANative.RandomJOVGenerator(
0, context).symbolStrings(self.symbol_strings)
for jov in jovs:
for i in range(100):
random_value = randomJOVGenerator.randomValue(jov)
if random_value != None:
self.assertTrue(jov.covers(random_value),
"jov %s doesn't cover %s, but generated it as a random value!" %
(jov, random_value)
)
def test_teardown_during_vector_load(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager=vdm)
context.evaluate(
FORA.extractImplValContainer(
FORA.eval("fun() { let v = [1,2,3].paged; fun() { v[1] } }")),
FORANative.symbol_Call)
vdm.unloadAllPossible()
pagedVecAccessFun = context.getFinishedResult().asResult.result
context.teardown()
context.evaluate(pagedVecAccessFun, FORANative.symbol_Call)
self.assertFalse(context.isInterrupted())
self.assertTrue(context.isVectorLoad())
context.teardown()
def createWorker(machineId, viewFactory, callbackSchedulerToUse = None, threadCount = 2, memoryLimitMb = 100):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
vdm = ForaNative.VectorDataManager(callbackSchedulerToUse, 5 * 1024 * 1024)
vdm.setMemoryLimit(
int(memoryLimitMb * 1024 * 1024),
min(int(memoryLimitMb * 1.25 * 1024 * 1024),
int((memoryLimitMb + 1024 * 2) * 1024 * 1024))
)
vdm.setPersistentCacheIndex(
CumulusNative.PersistentCacheIndex(
viewFactory.createView(),
callbackSchedulerToUse
)
)
cache = CumulusNative.SimpleOfflineCache(callbackSchedulerToUse, 1000 * 1024 * 1024)
eventHandler = CumulusNative.CumulusWorkerHoldEventsInMemoryEventHandler()
return (
CumulusNative.CumulusWorker(
callbackSchedulerToUse,
CumulusNative.CumulusWorkerConfiguration(
machineId,
threadCount,
CumulusNative.CumulusCheckpointPolicy.None(),
ExecutionContext.createContextConfiguration(),
""
),
vdm,
cache,
eventHandler
),
vdm,
eventHandler
)
def createCliqueFinderContext(self, text, *args):
maxPageSizeInBytes = 32 * 1024
vdm = FORANative.VectorDataManager(callbackScheduler,
maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager=vdm)
argumentImplvals = []
for a in args:
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { " + a + " }")),
Symbol_Call)
argumentImplvals.append(
context.getFinishedResult().asResult.result)
actualFunction = FORA.extractImplValContainer(FORA.eval(text))
context.placeInEvaluationState(
FORANative.ImplValContainer((actualFunction, Symbol_Call) +
tuple(argumentImplvals)))
return context, argumentImplvals, vdm
def simpleEval(*args, **kwargs):
return ExecutionContext.simpleEval(callbackScheduler, *args, **kwargs)
def simpleEval(*args, **kwargs):
return ExecutionContext.simpleEval(callbackScheduler, *args, **kwargs)
def __init__(self,
ownAddress,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnosticsDir,
config,
viewFactory):
Stoppable.Stoppable.__init__(self)
#acquire a machineId randomly, using uuid
self.machineId = CumulusNative.MachineId(
Hash.Hash.sha1(str(uuid.uuid4()))
)
self.ownAddress = ownAddress
self.callbackScheduler = callbackScheduler
self.viewFactory = viewFactory
self.threadsStarted_ = False
self.connectedMachines = set()
self.connectingMachines = set() # machines we are in the process of connecting to
self.droppedMachineIds = set()
self.lock = threading.RLock()
self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc
self.reconnectPersistentCacheIndexViewThreads = []
if config.cumulusDiskCacheStorageSubdirectory is not None:
self.cumulusDiskCacheWantsDeletionOnTeardown = True
self.cumulusDiskCacheStorageDir = os.path.join(
config.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageSubdirectory
)
else:
self.cumulusDiskCacheWantsDeletionOnTeardown = False
self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir
logging.info(
"Creating a CumulusService with ram cache of %s / %s MB and %s threads",
self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusThreadCountOverride
)
self._stopEvent = threading.Event()
self._channelListener = channelListener
assert len(self._channelListener.ports) == 2
self._channelFactory = channelFactory
Runtime.initialize()
ModuleImporter.initialize()
self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
self.viewFactory
)
self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
target=self._channelListener.start
)
self.vdm = VectorDataManager.constructVDM(
callbackScheduler,
self.cumulusVectorRamCacheSizeOverride,
self.cumulusMaxRamCacheSizeOverride
)
if self.cumulusTrackTcMalloc:
logging.info(
"CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0
)
self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()
self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
viewFactory.createView(retrySeconds=10.0, numRetries=10),
callbackScheduler
)
self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)
self.deleteCumulusDiskCacheIfNecessary()
self.offlineCache = CumulusNative.DiskOfflineCache(
callbackScheduler,
self.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageMB * 1024 * 1024,
config.cumulusDiskCacheStorageFileCount
)
checkpointInterval = config.cumulusCheckpointIntervalSeconds
if checkpointInterval == 0:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
else:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
checkpointInterval,
1024 * 1024
)
self.cumulusWorker = self.constructCumlusWorker(
callbackScheduler,
CumulusNative.CumulusWorkerConfiguration(
self.machineId,
self.cumulusThreadCountOverride,
checkpointPolicy,
ExecutionContext.createContextConfiguration(),
diagnosticsDir or ""
),
self.vdm,
self.offlineCache,
eventHandler
)
#externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
#callbackScheduler
#)
#self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
#settings.s3InterfaceFactory,
#settings.objectStore,
#self.vdm,
#externalDatasetChannel.makeQueuelike(callbackScheduler)
#)
self.cumulusWorker.startComputations()
