def test_tupleExpand_nodes_1(self):
text = "fun(h) { let (f, g) = h(); (f(), g()) }"
cfg = self.parseStringToFunction(text).toCFG(1)
hImplVal = FORA.extractImplValContainer(
FORA.eval(
"fun() { let f = fun() { 1 }; let g = fun() { 2 }; (f, g) }"
)
)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (hImplVal,))
res = evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
cfgWithFutures.slotCompleted(
0, res
)
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
submittableArgs = cfgWithFutures.submittableArgs(1)
cfgWithFutures.slotCompleted(
1, evalSubmittableArgs(submittableArgs)
)
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2])
# this verifies that parallelism is possible for `text`
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2, 3])
"""
def test_teardown_simple(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInternalSplitting=False)
evaluate(context,
FORA.extractImplValContainer(FORA.eval("fun(){nothing}")),
FORANative.symbol_Call)
context.getFinishedResult()
toEval = FORA.extractImplValContainer(
FORA.eval("""fun() {
let f = fun() { };
let v = [1, [3]];
cached(f())
}"""))
evaluate(context, toEval, FORANative.symbol_Call)
while not context.isCacheRequest():
context.compute()
context.teardown(True)
def createCliqueFinderContext(self, text, *args):
maxPageSizeInBytes = 32 * 1024
vdm = FORANative.VectorDataManager(callbackScheduler, maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm
)
argumentImplvals = []
for a in args:
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { " + a + " }")),
Symbol_Call
)
argumentImplvals.append(
context.getFinishedResult().asResult.result
)
actualFunction = FORA.extractImplValContainer(
FORA.eval(text)
)
context.placeInEvaluationState(
FORANative.ImplValContainer(
(actualFunction, Symbol_Call) + tuple(argumentImplvals)
)
)
return context, argumentImplvals, vdm
def submitComputationOnClient(self, clientIndex, expressionText, **freeVariables):
if isinstance(expressionText, ForaNative.ImplValContainer):
computationDefinition = CumulusNative.ComputationDefinition.Root(
CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm(
[CumulusNative.ComputationDefinitionTerm.Value(x, None) for x in expressionText.getTuple()]
)
)
elif isinstance(expressionText, CumulusNative.ComputationDefinition):
computationDefinition = expressionText
else:
varNames = list(freeVariables.keys())
expr = FORA.eval("fun(" + ",".join(varNames) + ") {" + expressionText + " } ")
computationDefinition = createComputationDefinition(
FORA.extractImplValContainer(
expr
),
ForaNative.makeSymbol("Call"),
*[freeVariables[v] for v in varNames]
)
computationId = self.getClient(clientIndex).createComputation(computationDefinition)
self.getClient(clientIndex).setComputationPriority(
computationId,
CumulusNative.ComputationPriority(1)
)
return computationId
def test_getObjectMember_4(self):
obj = FORA.extractImplValContainer(
FORA.eval("object { x: fun () { } }")
)
obj2 = obj.getObjectMember('x')
self.assertTrue(obj2 is not None)
def test_futures_1(self):
cfg = self.parseStringToFunction("fun(f) { f(1) + f(2) }").toCFG(1)
funImplval = FORA.extractImplValContainer(
FORA.eval("fun(x) { x + 1 }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, "block_0Let", (funImplval, ))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Let")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
asSubmittable = cfgWithFutures.submittableArgs(0)
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_1Let")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])
self.assertIsNotNone(cfgWithFutures.submittableArgs(1))
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_2Try")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])
self.assertIsNone(cfgWithFutures.submittableArgs(2))
self.assertIsNone(cfgWithFutures.submittableArgs(3))
def copyDataOutOfPagesTest(self, text, cycleCount, expectsToHaveCopies):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm,
allowInterpreterTracing=False,
allowInternalSplitting=False)
context.configuration.agressivelyValidateRefcountsAndPageReachability = True
context.configuration.releaseVectorHandlesImmediatelyAfterExecution = False
context.placeInEvaluationState(
FORANative.ImplValContainer(
(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)))
context.interruptAfterCycleCount(cycleCount)
context.compute()
if expectsToHaveCopies:
self.assertTrue(context.copyValuesOutOfVectorPages())
self.assertFalse(context.copyValuesOutOfVectorPages())
else:
self.assertFalse(context.copyValuesOutOfVectorPages())
def test_splitComputation_1(self):
text = """fun() {
let s = fun(a,b,f) {
if (b <= a)
return nothing
if (b <= a + 1)
return f(a)
let mid = Int64((a+b)/2)
return s(a,mid,f) + s(mid,b,f)
}
let m = [0];
s(0,1000,fun(x){m[0] * x})
}"""
funImplval = FORA.extractImplValContainer(FORA.eval(text))
# minimum value to split seems to be 40
pausedComputation = callAndExtractPausedCompuationAfterSteps(
funImplval, 40)
splitComputation = ForaNative.splitPausedComputation(pausedComputation)
self.assertTrue(splitComputation is not None)
def test_switch_nodes_1(self):
text = "fun(x, f, g) { match (x) with (1) { f(x) } (2) { g(x) } }"
cfg = self.parseStringToFunction(text).toCFG(3)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
gImplVal = FORA.extractImplValContainer(
FORA.eval("fun(x) { throw x }"))
xImplVal = ForaNative.ImplValContainer(1)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (xImplVal, fImplval, gImplVal))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Try")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_6Try")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0)))
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(2))
def test_mutable_values_2(self):
text = "fun(f, g) { let x = f(); let y = g(); 1 }"
cfg = self.parseStringToFunction(text).toCFG(2)
fImplval = FORA.extractImplValContainer(
FORA.eval("let f = fun() { Vector.range(10) }; f"))
gImplval = FORA.extractImplValContainer(
FORA.eval(
"let g = fun() { MutableVector(Float64).create(10, 0.0) }; g"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (fImplval, gImplval))
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
self.assertFalse(cfgWithFutures.hasResolvedToSimpleState())
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])
cfgWithFutures.slotCompleted(
1, evalSubmittableArgs(cfgWithFutures.submittableArgs(1)))
self.assertTrue(cfgWithFutures.mustBailEarly())
self.assertFalse(cfgWithFutures.hasResolvedToSimpleState())
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0)))
self.assertTrue(cfgWithFutures.mustBailEarly())
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(1))
def evalApplyTuple(applyTuple, signature=None):
varNames = map(lambda ix: "x_%s" % ix, range(len(applyTuple)))
if signature is None:
evalStringTerms = varNames
else:
evalStringTerms = []
for ix in range(len(signature.terms)):
term = signature.terms[ix]
if term.isNormal():
name = term.asNormal.name
if name is not None:
evalStringTerms.append("%s: x_%s" % (name, ix))
else:
evalStringTerms.append("x_%s" % ix)
elif term.isTupleCall():
evalStringTerms.append("*x_%s" % ix)
evalString = evalStringTerms[0] + "`(" + ",".join(
evalStringTerms[1:]) + ")"
try:
return normalComputationResult(
FORA.extractImplValContainer(
FORA.eval(evalString,
locals={
name: val
for (name, val) in zip(varNames, applyTuple)
},
keepAsForaValue=True)))
except ForaValue.FORAException as e:
return exceptionComputationResult(
FORA.extractImplValContainer(e.foraVal))
def test_futures_with_branching_1(self):
cfg = self.parseStringToFunction(
"fun(x, f, g) { if (x) return f(x) else g(x) }").toCFG(3)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
gImplVal = FORA.extractImplValContainer(
FORA.eval("fun(x) { throw x }"))
xImplVal = ForaNative.ImplValContainer(1)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (xImplVal, fImplval, gImplVal))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Branch")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_1Return")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0)))
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(2))
def test_cached_nodes_1(self):
text = "fun(f, g) { cached(f(), g()); }"
cfg = self.parseStringToFunction(text).toCFG(2)
fImplval = FORA.extractImplValContainer(FORA.eval("fun() { 1 }"))
gImplVal = FORA.extractImplValContainer(FORA.eval("fun() { 2 }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (fImplval, gImplVal))
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
res = evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
expectedResult = ForaNative.ImplValContainer((1, 2))
self.assertEqual(res.asResult.result, expectedResult)
cfgWithFutures.slotCompleted(0, res)
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
expectedResult
)
def test_future_fuzz_5(self):
cfg = self.parseStringToFunction(
"fun(f) { f(1) + f(2) + f(3) + f(4) }").toCFG(1)
nodeValues = (FORA.extractImplValContainer(
FORA.eval("fun(x) { if (x <= 2) throw x else x + 1 }")), )
self.fuzz(cfg, nodeValues, 10)
def test_stringify(self):
array = FORA.extractImplValContainer(FORA.eval("[1,2,3.4]"))
stringified = self.evaluator.getVDM().stringifyAndJoinVectorContents(
array, ",", 0, array.getVectorSize())
self.assertEqual(stringified, "1,2,3.4")
def test_expensive_brownian(self):
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let brownian = fun(x,t) {
if (t == 0)
return sum(0.0, x * 10.0**7.0)
else
{
let (l,r) = cached(brownian(x - 1, t - 1), brownian(x + 1, t - 1));
return sum(0.0, x * 10.0**7.0) + l + r
}
}
brownian(0, 5)
}"""
)
)
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
self.assertTrue(response[1].isResult())
def test_verifyThatExtractingPausedComputationsDoesntDuplicateLargeStrings(
self):
text = """fun() {
let s = ' '
while (size(s) < 1000000)
s = s + s
let f = fun(x) { if (x > 0) return f(x-1) + s[x]; `TriggerInterruptForTesting() }
f(20)
}"""
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
computation = context.extractPausedComputation()
context2 = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context2.resumePausedComputation(computation)
self.assertTrue(context2.totalBytesUsed < 2 * context.totalBytesUsed)
def test_extractPausedComputationDuringVectorLoad(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call")))
pagedVec = context.getFinishedResult().asResult.result
context.placeInEvaluationState(
FORANative.ImplValContainer(
(pagedVec,
FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
FORANative.ImplValContainer(0))))
vdm.unloadAllPossible()
context.resume()
self.assertTrue(context.isVectorLoad())
computation = context.extractPausedComputation()
self.assertEqual(len(computation.frames), 1)
def test_resumingAfterCopyDataOutOfPages(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager=vdm)
context.interruptAfterCycleCount(100000)
text = """
fun() {
let v = Vector.range(1000).paged;
let ix1 = 0
let res = 0
while (true) {
res = res + v[ix1]
ix1 = (ix1 + 1) % size(v)
}
res
}"""
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
paused1 = context.extractPausedComputation()
while not context.isVectorLoad():
context.copyValuesOutOfVectorPages()
vdm.unloadAllPossible()
context.interruptAfterCycleCount(100000)
context.resume()
paused2 = context.extractPausedComputation()
self.assertTrue(len(paused1.frames) == len(paused2.frames))
def test_refcountsInCompiledCode(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm,
allowInterpreterTracing=True,
blockUntilTracesAreCompiled=True)
text = """fun(){
let f = fun(v, depth) {
if (depth > 100)
//this will trigger an interrupt since the data cannot exist in the VDM
datasets.s3('','')
else
f(v, depth+1)
}
f([1,2,3,4,5], 0)
}"""
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
stacktraceText = context.extractCurrentTextStacktrace()
self.assertTrue(stacktraceText.count("Vector") < 10)
def test_sortManySmallVectors(self):
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let shouldAllBeTrue =
Vector.range(20, fun(o) {
sorting.isSorted(
sort(Vector.range(50000 + o, fun(x) { x / 10 }))
)
});
for s in shouldAllBeTrue {
if (not s)
return false
}
return true
}"""
)
)
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == True, response[1].asResult.result.pyval)
def test_teardown_simple(self):
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager = vdm)
context.evaluate(
FORA.extractImplValContainer(
FORA.eval("fun(){nothing}")
),
FORANative.symbol_Call
)
context.getFinishedResult()
toEval = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let f = fun() { };
let v = [1, [3]];
cached(f())
}"""
)
)
context.evaluate(toEval, FORANative.symbol_Call)
while not context.isCacheRequest():
context.resume()
context.teardown(True)
def test_manyDuplicateCachecallsAndAdding(self):
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
Vector.range(20).papply(fun(x) {
x + cached(sum(0,10**11))[0]
})
}"""
)
)
computationId = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
time.sleep(5)
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
try:
response = self.gateway.finalResponses.get(timeout=240.0)
except Queue.Empty:
response = None
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.gateway.deprioritizeComputation(computationId)
def test_bigSumWithAdding(self):
self.desirePublisher.desireNumberOfWorkers(1, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval("""fun() { if (sum(0, 10**11) > 0) 'big_enough' }"""))
computationId = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(expr,
ForaNative.makeSymbol("Call")))
time.sleep(5)
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
try:
response = self.gateway.finalResponses.get(timeout=240.0)
except Queue.Empty:
response = None
if response is None:
try:
dumpFun = self.dumpSchedulerEventStreams
dumpFun()
except:
logging.warn(
"Wanted to dump CumulusWorkerEvents, but couldn't")
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == 'big_enough',
response[1].asResult.result.pyval)
self.gateway.deprioritizeComputation(computationId)
def test_refcountsInCompiledCode(self):
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = True,
blockUntilTracesAreCompiled = True
)
text = """fun(){
let f = fun(v, depth) {
if (depth > 100)
//this will trigger an interrupt since the data cannot exist in the VDM
datasets.s3('','')
else
f(v, depth+1)
}
f([1,2,3,4,5], 0)
}"""
context.evaluate(
FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call
)
stacktraceText = context.extractCurrentTextStacktrace()
self.assertTrue(stacktraceText.count("Vector") < 10)
def test_serialize_while_holding_interior_vector(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm,
allowInterpreterTracing=False,
allowInternalSplitting=False)
evaluate(
context,
FORA.extractImplValContainer(
FORA.eval("""
fun() {
let v = [[1].paged].paged;
let v2 = v[0]
`TriggerInterruptForTesting()
1+2+3+v+v2
}""")), FORANative.symbol_Call)
self.assertTrue(context.isInterrupted())
serialized = context.serialize()
context = None
def test_verifyThatExtractingPausedComputationsDoesntDuplicateLargeStrings(self):
text = """fun() {
let s = ' '
while (size(s) < 1000000)
s = s + s
let f = fun(x) { if (x > 0) return f(x-1) + s[x]; `TriggerInterruptForTesting() }
f(20)
}"""
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
context.evaluate(
FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call
)
computation = context.extractPausedComputation()
context2 = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
context2.resumePausedComputation(computation)
self.assertTrue(
context2.totalBytesUsed < 2 * context.totalBytesUsed
)
def test_getObjectMember_3(self):
obj = FORA.extractImplValContainer(
FORA.eval("object { x: object { y: 20 } }")
)
obj2 = obj.getObjectMember('x')
self.assertEqual(obj2.getObjectMember('y').pyval, 20)
def test_extractPausedComputationDuringVectorLoad(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
)
pagedVec = context.getFinishedResult().asResult.result
context.placeInEvaluationState(
FORANative.ImplValContainer(
(pagedVec,
FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
FORANative.ImplValContainer(0))
)
)
vdm.unloadAllPossible()
context.resume()
self.assertTrue(context.isVectorLoad())
computation = context.extractPausedComputation()
self.assertEqual(len(computation.frames),1)
def test_vecWithinVec(self):
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let v =
Vector.range(10) ~~ fun(x) {
Vector.range(1000000 + x).paged
};
v = v.paged;
let res = ()
for elt in v
res = res + (elt,)
res..apply(fun(tupElt) { tupElt.sum() })
'got to the end'
}"""
)
)
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == 'got to the end', response[1].asResult.result.pyval)
def test_sortALargeVectorWithFourWorkers(self):
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let v = Vector.range(12500000, {_%3.1415926});
if (sorting.isSorted(sorting.sort(v))) 'sorted'
}"""
)
)
try:
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
),
timeout=120.0
)
except Queue.Empty:
response = None
if response is None:
try:
dumpFun = self.dumpSchedulerEventStreams
dumpFun()
except:
logging.warn("Wanted to dump CumulusWorkerEvents, but couldn't");
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == 'sorted', response[1].asResult.result.pyval)
def test_futures_with_branching_2(self):
text = "fun(x, f) { let res = f(x); if (res) 0 else 1 }"
cfg = self.parseStringToFunction(text).toCFG(2)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
xImplVal = ForaNative.ImplValContainer(1)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (xImplVal, fImplval))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Let")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0)))
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_1Branch")
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
cfgWithFutures.continueSimulation()
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(0))
def test_serialize_while_holding_interior_vector(self):
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager = vdm, allowInterpreterTracing=False)
context.evaluate(
FORA.extractImplValContainer(
FORA.eval("""
fun() {
let v = [[1].paged].paged;
let v2 = v[0]
`TriggerInterruptForTesting()
1+2+3+v+v2
}"""
)
),
FORANative.symbol_Call
)
self.assertTrue(context.isInterrupted())
serialized = context.serialize()
context = None
def test_tupleExpand_nodes_1(self):
text = "fun(h) { let (f, g) = h(); (f(), g()) }"
cfg = self.parseStringToFunction(text).toCFG(1)
hImplVal = FORA.extractImplValContainer(
FORA.eval(
"fun() { let f = fun() { 1 }; let g = fun() { 2 }; (f, g) }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (hImplVal, ))
res = evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
cfgWithFutures.slotCompleted(0, res)
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
submittableArgs = cfgWithFutures.submittableArgs(1)
cfgWithFutures.slotCompleted(1, evalSubmittableArgs(submittableArgs))
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2])
# this verifies that parallelism is possible for `text`
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2, 3])
"""
def test_teardown_during_vector_load(self):
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager = vdm)
context.evaluate(
FORA.extractImplValContainer(
FORA.eval("fun() { let v = [1,2,3].paged; fun() { v[1] } }")
),
FORANative.symbol_Call
)
vdm.unloadAllPossible()
pagedVecAccessFun = context.getFinishedResult().asResult.result
context.teardown()
context.evaluate(
pagedVecAccessFun,
FORANative.symbol_Call
)
self.assertFalse(context.isInterrupted())
self.assertTrue(context.isVectorLoad())
context.teardown()
def test_future_fuzz_5(self):
cfg = self.parseStringToFunction(
"fun(f) { f(1) + f(2) + f(3) + f(4) }").toCFG(1)
nodeValues = (FORA.extractImplValContainer(
FORA.eval("fun(x) { if (x <= 2) throw x else x + 1 }")),)
self.fuzz(cfg, nodeValues, 10)
def test_vectorApplyWithAdding(self):
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval("""fun() {
let v = [1,2,3,4].paged;
let res = 0
sum(0,20000000000, fun(x) { v[x%4] })
'test_vectorApplyWithAdding'
}"""))
computationId = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(expr,
ForaNative.makeSymbol("Call")))
try:
response = self.gateway.finalResponses.get(timeout=5.0)
except Queue.Empty:
response = None
self.assertTrue(response is None, response)
self.desirePublisher.desireNumberOfWorkers(3, blocking=True)
response = self.gateway.finalResponses.get(timeout=240.0)
self.gateway.deprioritizeComputation(computationId)
self.assertTrue(response[1].isResult())
self.assertTrue(
response[1].asResult.result.pyval == 'test_vectorApplyWithAdding',
response[1].asResult.result.pyval)
def test_switch_nodes_1(self):
text = "fun(x, f, g) { match (x) with (1) { f(x) } (2) { g(x) } }"
cfg = self.parseStringToFunction(text).toCFG(3)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
gImplVal = FORA.extractImplValContainer(FORA.eval("fun(x) { throw x }"))
xImplVal = ForaNative.ImplValContainer(1)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (xImplVal, fImplval, gImplVal))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Try")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_6Try")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
)
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(2)
)
def test_futures_7(self):
funString = "fun(f, x) { let res = f(x, *x); res + 1 }"
cfg = self.parseStringToFunction(funString).toCFG(2)
fImplval = FORA.extractImplValContainer(
FORA.eval("fun(*x) { size(x) }"))
xImplVal = ForaNative.ImplValContainer((1, 2, 3))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, "block_0Let", (fImplval, xImplVal))
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.submittableArgs(0)
cfgWithFutures.slotCompleted(
0, normalComputationResult(ForaNative.ImplValContainer(4)))
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [1])
cfgWithFutures.slotCompleted(
1, normalComputationResult(ForaNative.ImplValContainer(5)))
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
cfgWithFutures.continueSimulation()
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(5))
def test_futures_with_branching_2(self):
text = "fun(x, f) { let res = f(x); if (res) 0 else 1 }"
cfg = self.parseStringToFunction(text).toCFG(2)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
xImplVal = ForaNative.ImplValContainer(1)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (xImplVal, fImplval))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Let")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
)
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_1Branch")
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
cfgWithFutures.continueSimulation()
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(0)
)
def test_futures_with_branching_1(self):
cfg = self.parseStringToFunction(
"fun(x, f, g) { if (x) return f(x) else g(x) }").toCFG(3)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
gImplVal = FORA.extractImplValContainer(FORA.eval("fun(x) { throw x }"))
xImplVal = ForaNative.ImplValContainer(1)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (xImplVal, fImplval, gImplVal))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Branch")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_1Return")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
)
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(2)
)
def test_futures_7(self):
funString = "fun(f, x) { let res = f(x, *x); res + 1 }"
cfg = self.parseStringToFunction(funString).toCFG(2)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(*x) { size(x) }"))
xImplVal = ForaNative.ImplValContainer((1,2,3));
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, "block_0Let", (fImplval, xImplVal))
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.submittableArgs(0)
cfgWithFutures.slotCompleted(
0, normalComputationResult(ForaNative.ImplValContainer(4))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [1])
cfgWithFutures.slotCompleted(
1, normalComputationResult(ForaNative.ImplValContainer(5))
)
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
cfgWithFutures.continueSimulation()
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(5)
)
def evalApplyTuple(applyTuple, signature = None):
varNames = map(lambda ix: "x_%s" % ix, range(len(applyTuple)))
if signature is None:
evalStringTerms = varNames
else:
evalStringTerms = []
for ix in range(len(signature.terms)):
term = signature.terms[ix]
if term.isNormal():
name = term.asNormal.name
if name is not None:
evalStringTerms.append("%s: x_%s" % (name, ix))
else:
evalStringTerms.append("x_%s" % ix)
elif term.isTupleCall():
evalStringTerms.append("*x_%s" % ix)
evalString = evalStringTerms[0] + "`(" + ",".join(evalStringTerms[1:]) + ")"
try:
return normalComputationResult(
FORA.extractImplValContainer(
FORA.eval(
evalString,
locals = { name: val
for (name, val) in
zip(varNames, applyTuple) },
keepAsForaValue = True
)
)
)
except ForaValue.FORAException as e:
return exceptionComputationResult(
FORA.extractImplValContainer(e.foraVal)
)
def test_futures_4(self):
funString = "fun(f) { f(0) + (f(1) * f(2)) }"
cfg = self.parseStringToFunction(funString).toCFG(1)
funImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, "block_0Let", (funImplval,))
for ix in range(3):
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1, 2])
for ix in range(3):
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1, 2])
cfgWithFutures.submittableArgs(2)
cfgWithFutures.slotCompleted(
2, normalComputationResult(ForaNative.ImplValContainer(2))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])
cfgWithFutures.submittableArgs(1)
cfgWithFutures.slotCompleted(
1, normalComputationResult(ForaNative.ImplValContainer(1))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 3])
cfgWithFutures.submittableArgs(3)
cfgWithFutures.slotCompleted(
3, normalComputationResult(ForaNative.ImplValContainer(2))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.submittableArgs(0)
cfgWithFutures.slotCompleted(
0, normalComputationResult(ForaNative.ImplValContainer(0))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [4])
cfgWithFutures.submittableArgs(4)
cfgWithFutures.slotCompleted(
4, normalComputationResult(ForaNative.ImplValContainer(2))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
self.assertEqual(cfgWithFutures.currentLabel(), "block_2Try")
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(2)
)
def test_futures_4(self):
funString = "fun(f) { f(0) + (f(1) * f(2)) }"
cfg = self.parseStringToFunction(funString).toCFG(1)
funImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, "block_0Let", (funImplval,))
for ix in range(3):
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1, 2])
for ix in range(3):
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1, 2])
cfgWithFutures.submittableArgs(2)
cfgWithFutures.slotCompleted(
2, normalComputationResult(ForaNative.ImplValContainer(2))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])
cfgWithFutures.submittableArgs(1)
cfgWithFutures.slotCompleted(
1, normalComputationResult(ForaNative.ImplValContainer(1))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 3])
cfgWithFutures.submittableArgs(3)
cfgWithFutures.slotCompleted(
3, normalComputationResult(ForaNative.ImplValContainer(2))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.submittableArgs(0)
cfgWithFutures.slotCompleted(
0, normalComputationResult(ForaNative.ImplValContainer(0))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [4])
cfgWithFutures.submittableArgs(4)
cfgWithFutures.slotCompleted(
4, normalComputationResult(ForaNative.ImplValContainer(2))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
self.assertEqual(cfgWithFutures.currentLabel(), "block_2Try")
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(2)
)
def test_getObjectDefinitionTermsWithMetadata_1(self):
obj = FORA.extractImplValContainer(
FORA.eval('object { "x!!!" x: 10; "operator+!!" operator +(other) { 10 }; "F!" f: fun(){}; g: fun(){} };')
)
objTermsWithMeta = obj.getObjectDefinitionTermsWithMetadata()
#TODO have a better test here
self.assertEqual(len(objTermsWithMeta), 4)
def test_basic_gpu_codegen(self):
f = Fora.extractImplValContainer(
Fora.eval("fun(e) { `LocalityHint((e,e+1)); e + 1 }"))
v = Fora.extractImplValContainer(Fora.eval("[1,2,3,4]"))
#this is a very weak way of testing this, but works as a first pass
self.assertTrue("LocalityHint" in
ForaNative.compileAndStringifyNativeCfgForGpu(f, v))
def memberFromString(self, foraText, member):
obj = FORA.extractImplValContainer(FORA.eval(foraText))
res = obj.getObjectMember(member)
if res is None:
return None
return FORA.ForaValue.FORAValue(res).toPythonObject()
def test_getObjectDefinitionTermsWithMetadata_2(self):
obj = FORA.extractImplValContainer(
FORA.eval('class { "x!" member x; "new!" operator new() {}; "F!" f: fun() {}; "static!" static g: "inner" fun(){}; "operator+" operator+(){}; }')
)
objTermsWithMeta = obj.getObjectDefinitionTermsWithMetadata()
#TODO have a better test here
self.assertEqual(len(objTermsWithMeta), 5)
def test_future_fuzz_6(self):
funString = "fun(f, x) { let res = f(x, *x); res + 1 }"
cfg = self.parseStringToFunction(funString).toCFG(2)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(*x) { size(x) }"))
xImplVal = ForaNative.ImplValContainer((1,2,3));
nodeValues = (fImplval, xImplVal)
self.fuzz(cfg, nodeValues, 10)
def createComputedValue(*strings):
return ComputedValue.ComputedValue(
args=tuple([
(FORA.extractImplValContainer(
ForaValue.FORAValue(FORA.eval(x))
) if isinstance(x, str) else x)
for x in strings
])
)
def test_resumePausedComputationWithResult(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
text = """
let f = fun(v, ix) {
if (ix > 0)
{
let (v2,res) = f(v,ix-1);
return (v2, res + v2[0])
}
`TriggerInterruptForTesting()
return (v, 0)
};
f([1], 10)
"""
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
)
assert context.isInterrupted()
pausedComp = context.extractPausedComputation()
framesToUse = pausedComp.frames[0:5]
pausedComp2 = FORANative.PausedComputation(
framesToUse,
FORA.extractImplValContainer(FORA.eval("([2], 0)", keepAsForaValue=True)),
False
)
context.resumePausedComputation(pausedComp2)
context.copyValuesOutOfVectorPages()
context.pageLargeVectorHandles(0)
context.resetInterruptState()
context.resume()
self.assertTrue( context.isFinished() )
result = context.getFinishedResult()
self.assertTrue(result.asResult.result[1].pyval == 6)
def numpytest(text, dtype, expr):
array = FORA.extractImplValContainer(FORA.eval(text))
nArray = self.evaluator.getVDM().extractVectorContentsAsNumpyArray(array, 0, array.getVectorSize())
self.assertTrue(
(nArray == expr.astype('int64')).all(),
"Expected %s, got %s" % (expr.astype('int64'), nArray)
)