def test_switch_nodes_1(self):
text = "fun(x, f, g) { match (x) with (1) { f(x) } (2) { g(x) } }"
cfg = self.parseStringToFunction(text).toCFG(3)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
gImplVal = FORA.extractImplValContainer(FORA.eval("fun(x) { throw x }"))
xImplVal = ForaNative.ImplValContainer(1)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (xImplVal, fImplval, gImplVal))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Try")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_6Try")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
)
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(2)
)
def test_futures_with_branching_1(self):
cfg = self.parseStringToFunction(
"fun(x, f, g) { if (x) return f(x) else g(x) }").toCFG(3)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
gImplVal = FORA.extractImplValContainer(FORA.eval("fun(x) { throw x }"))
xImplVal = ForaNative.ImplValContainer(1)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (xImplVal, fImplval, gImplVal))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Branch")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_1Return")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
)
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(2)
)
def test_cached_nodes_1(self):
text = "fun(f, g) { cached(f(), g()); }"
cfg = self.parseStringToFunction(text).toCFG(2)
fImplval = FORA.extractImplValContainer(FORA.eval("fun() { 1 }"))
gImplVal = FORA.extractImplValContainer(FORA.eval("fun() { 2 }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (fImplval, gImplVal))
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
res = evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
expectedResult = ForaNative.ImplValContainer((1, 2))
self.assertEqual(res.asResult.result, expectedResult)
cfgWithFutures.slotCompleted(0, res)
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
expectedResult
)
def createCliqueFinderContext(self, text, *args):
maxPageSizeInBytes = 32 * 1024
vdm = FORANative.VectorDataManager(callbackScheduler, maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm
)
argumentImplvals = []
for a in args:
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { " + a + " }")),
Symbol_Call
)
argumentImplvals.append(
context.getFinishedResult().asResult.result
)
actualFunction = FORA.extractImplValContainer(
FORA.eval(text)
)
context.placeInEvaluationState(
FORANative.ImplValContainer(
(actualFunction, Symbol_Call) + tuple(argumentImplvals)
)
)
return context, argumentImplvals, vdm
def test_teardown_simple(self):
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager = vdm)
context.evaluate(
FORA.extractImplValContainer(
FORA.eval("fun(){nothing}")
),
FORANative.symbol_Call
)
context.getFinishedResult()
toEval = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let f = fun() { };
let v = [1, [3]];
cached(f())
}"""
)
)
context.evaluate(toEval, FORANative.symbol_Call)
while not context.isCacheRequest():
context.resume()
context.teardown(True)
def test_resumePausedComputationWithResult(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
text = """
let f = fun(v, ix) {
if (ix > 0)
{
let (v2,res) = f(v,ix-1);
return (v2, res + v2[0])
}
`TriggerInterruptForTesting()
return (v, 0)
};
f([1], 10)
"""
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
)
assert context.isInterrupted()
pausedComp = context.extractPausedComputation()
framesToUse = pausedComp.frames[0:5]
pausedComp2 = FORANative.PausedComputation(
framesToUse,
FORA.extractImplValContainer(FORA.eval("([2], 0)", keepAsForaValue=True)),
False
)
context.resumePausedComputation(pausedComp2)
context.copyValuesOutOfVectorPages()
context.pageLargeVectorHandles(0)
context.resetInterruptState()
context.resume()
self.assertTrue( context.isFinished() )
result = context.getFinishedResult()
self.assertTrue(result.asResult.result[1].pyval == 6)
def test_vectorOfPagedVectorApplyWithDropping(self):
self.desirePublisher.desireNumberOfWorkers(3, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let v =
Vector.range(20).apply(fun(ix) {
Vector.range(1250000.0+ix).paged
}).paged
let lookup = fun(ix) { v[ix] }
Vector.range(100).apply(fun(ix) { sum(0, 10**8); cached(lookup(ix)) })
v
}"""
)
)
computation1 = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
try:
response = self.gateway.finalResponses.get(timeout=60.0)
except Queue.Empty:
response = None
self.assertTrue(response is not None, response)
self.gateway.deprioritizeComputation(computation1)
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr2 = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let res = 0
for ix in sequence(10)
res = res + Vector.range(12500000+ix).sum()
return res
}"""
)
)
computation2 = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(
expr2,
ForaNative.makeSymbol("Call")
)
)
try:
response = self.gateway.finalResponses.get(timeout=60.0)
except Queue.Empty:
response = None
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.gateway.deprioritizeComputation(computation2)
def test_deep_parse_trees_trigger_ParseErrors(self):
sz = 1000
s = "0 + "
for i in range(sz - 1):
s = s + "1 + "
s = s + "1"
try:
Fora.eval(s)
self.assertTrue(False)
except ParseException:
self.assertTrue(True)
def test_serializationOfMutableVectors(self):
#verify that if we serialize two copies of the same vector that we get the same
#actual value back out
mutVecPair = FORA.eval("let x = MutableVector(Float64).create(10, 0.0); (x,x)", keepAsForaValue = True)
mutVecPairSer = FORA.extractImplValContainer(mutVecPair).serializeEntireObjectGraph()
mutVecPair2 = ivc()
mutVecPair2.deserializeEntireObjectGraph(mutVecPairSer)
mutVecPair2 = FORA.ForaValue.FORAValue(mutVecPair2)
self.assertTrue(FORA.eval("fun((x,y)) { x[0] = 1.0; y[0] is 1.0 }")(mutVecPair2))
self.assertTrue(FORA.eval("fun((x,y)) { x is y }")(mutVecPair2))
def test_throwUpdatesVariables(self):
locals = {}
try:
FORA.eval("let x = 10; throw 20", locals)
self.assertTrue(False)
except ForaValue.FORAException as e:
self.assertEqual(e.foraVal, 20)
self.assertTrue(e.trace is not None)
self.assertTrue(isinstance(e.trace, list))
for l in e.trace:
assert isinstance(l, HashNative.Hash)
self.assertEqual(locals, {'x':10})
def evalTimingsForExpr(expression, callResult = False):
if callResult:
toCall = FORA.eval(expression)
evalFun = lambda n: FORA.eval(
"toCall()",
locals = { "toCall" : toCall },
parsePath = ["LocalPerfTestRunner"]
)
else:
evalFun = lambda n: FORA.eval(
expression, parsePath = ["LocalPerfTestRunner"]
)
measureTimingsInLoop(evalFun)
def test_createInstance_2(self):
classIvc = FORA.eval(
"""let C = class {
member y;
member x;
};
C"""
).implVal_
x = 1
y = 2
res = ForaNative.simulateApply(
ForaNative.ImplValContainer(
(
classIvc,
makeSymbolIvc("CreateInstance"),
ForaNative.ImplValContainer(y),
ForaNative.ImplValContainer(x),
)
)
)
self.assertIsNotNone(res)
computed_x = ForaNative.simulateApply(
ForaNative.ImplValContainer((res, self.Symbol_member, makeSymbolIvc("x")))
)
self.assertEqual(computed_x, ForaNative.ImplValContainer(x))
def test_refcountsInCompiledCode(self):
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = True,
blockUntilTracesAreCompiled = True
)
text = """fun(){
let f = fun(v, depth) {
if (depth > 100)
//this will trigger an interrupt since the data cannot exist in the VDM
datasets.s3('','')
else
f(v, depth+1)
}
f([1,2,3,4,5], 0)
}"""
context.evaluate(
FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call
)
stacktraceText = context.extractCurrentTextStacktrace()
self.assertTrue(stacktraceText.count("Vector") < 10)
def test_sortALargeVectorWithFourWorkers(self):
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let v = Vector.range(12500000, {_%3.1415926});
if (sorting.isSorted(sorting.sort(v))) 'sorted'
}"""
)
)
try:
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
),
timeout=120.0
)
except Queue.Empty:
response = None
if response is None:
try:
dumpFun = self.dumpSchedulerEventStreams
dumpFun()
except:
logging.warn("Wanted to dump CumulusWorkerEvents, but couldn't");
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == 'sorted', response[1].asResult.result.pyval)
def test_vecWithinVec(self):
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let v =
Vector.range(10) ~~ fun(x) {
Vector.range(1000000 + x).paged
};
v = v.paged;
let res = ()
for elt in v
res = res + (elt,)
res..apply(fun(tupElt) { tupElt.sum() })
'got to the end'
}"""
)
)
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == 'got to the end', response[1].asResult.result.pyval)
def eval_expression(code, locals, showTimes=False, index = None):
evaluateStartTime = time.time()
try:
evalHistory.append(code)
result = FORA.eval(
code,
locals,
parsePath = ["EvalLoop", str(len(evalHistory) - 1)]
)
locals["result"] = result
handleEvalResult(result)
except ModuleImporter.FORAImportException as e:
handleEvalImportException(e)
except ParseException.ParseException as e:
handleEvalParseException(e)
except ForaValue.FORAFailure as e:
handleFailure(e.error)
except ForaValue.FORAException as e:
handleEvalForaException(e)
except Exceptions.ClusterException as e:
handleClusterException(e)
evaluateFinishTime = time.time()
if showTimes:
print "[eval time {0}]".format(
evaluateFinishTime - evaluateStartTime
)
def test_sortManySmallVectors(self):
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let shouldAllBeTrue =
Vector.range(20, fun(o) {
sorting.isSorted(
sort(Vector.range(50000 + o, fun(x) { x / 10 }))
)
});
for s in shouldAllBeTrue {
if (not s)
return false
}
return true
}"""
)
)
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == True, response[1].asResult.result.pyval)
def test_tupleExpand_nodes_1(self):
text = "fun(h) { let (f, g) = h(); (f(), g()) }"
cfg = self.parseStringToFunction(text).toCFG(1)
hImplVal = FORA.extractImplValContainer(
FORA.eval(
"fun() { let f = fun() { 1 }; let g = fun() { 2 }; (f, g) }"
)
)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (hImplVal,))
res = evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
cfgWithFutures.slotCompleted(
0, res
)
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
submittableArgs = cfgWithFutures.submittableArgs(1)
cfgWithFutures.slotCompleted(
1, evalSubmittableArgs(submittableArgs)
)
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2])
# this verifies that parallelism is possible for `text`
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2, 3])
"""
def test_future_fuzz_5(self):
cfg = self.parseStringToFunction(
"fun(f) { f(1) + f(2) + f(3) + f(4) }").toCFG(1)
nodeValues = (FORA.extractImplValContainer(
FORA.eval("fun(x) { if (x <= 2) throw x else x + 1 }")),)
self.fuzz(cfg, nodeValues, 10)
def test_getObjectMember_4(self):
obj = FORA.extractImplValContainer(
FORA.eval("object { x: fun () { } }")
)
obj2 = obj.getObjectMember('x')
self.assertTrue(obj2 is not None)
def test_teardown_during_vector_load(self):
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager = vdm)
context.evaluate(
FORA.extractImplValContainer(
FORA.eval("fun() { let v = [1,2,3].paged; fun() { v[1] } }")
),
FORANative.symbol_Call
)
vdm.unloadAllPossible()
pagedVecAccessFun = context.getFinishedResult().asResult.result
context.teardown()
context.evaluate(
pagedVecAccessFun,
FORANative.symbol_Call
)
self.assertFalse(context.isInterrupted())
self.assertTrue(context.isVectorLoad())
context.teardown()
def test_expensive_brownian(self):
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let brownian = fun(x,t) {
if (t == 0)
return sum(0.0, x * 10.0**7.0)
else
{
let (l,r) = cached(brownian(x - 1, t - 1), brownian(x + 1, t - 1));
return sum(0.0, x * 10.0**7.0) + l + r
}
}
brownian(0, 5)
}"""
)
)
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
self.assertTrue(response[1].isResult())
def test_verifyThatExtractingPausedComputationsDoesntDuplicateLargeStrings(self):
text = """fun() {
let s = ' '
while (size(s) < 1000000)
s = s + s
let f = fun(x) { if (x > 0) return f(x-1) + s[x]; `TriggerInterruptForTesting() }
f(20)
}"""
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
context.evaluate(
FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call
)
computation = context.extractPausedComputation()
context2 = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
context2.resumePausedComputation(computation)
self.assertTrue(
context2.totalBytesUsed < 2 * context.totalBytesUsed
)
def test_futures_with_branching_2(self):
text = "fun(x, f) { let res = f(x); if (res) 0 else 1 }"
cfg = self.parseStringToFunction(text).toCFG(2)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
xImplVal = ForaNative.ImplValContainer(1)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (xImplVal, fImplval))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Let")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
)
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_1Branch")
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
cfgWithFutures.continueSimulation()
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(0)
)
def test_extractPausedComputationDuringVectorLoad(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
)
pagedVec = context.getFinishedResult().asResult.result
context.placeInEvaluationState(
FORANative.ImplValContainer(
(pagedVec,
FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
FORANative.ImplValContainer(0))
)
)
vdm.unloadAllPossible()
context.resume()
self.assertTrue(context.isVectorLoad())
computation = context.extractPausedComputation()
self.assertEqual(len(computation.frames),1)
def test_getObjectMember_3(self):
obj = FORA.extractImplValContainer(
FORA.eval("object { x: object { y: 20 } }")
)
obj2 = obj.getObjectMember('x')
self.assertEqual(obj2.getObjectMember('y').pyval, 20)
def test_serialize_while_holding_interior_vector(self):
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager = vdm, allowInterpreterTracing=False)
context.evaluate(
FORA.extractImplValContainer(
FORA.eval("""
fun() {
let v = [[1].paged].paged;
let v2 = v[0]
`TriggerInterruptForTesting()
1+2+3+v+v2
}"""
)
),
FORANative.symbol_Call
)
self.assertTrue(context.isInterrupted())
serialized = context.serialize()
context = None
def test_manyDuplicateCachecallsAndAdding(self):
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
Vector.range(20).papply(fun(x) {
x + cached(sum(0,10**11))[0]
})
}"""
)
)
computationId = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
time.sleep(5)
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
try:
response = self.gateway.finalResponses.get(timeout=240.0)
except Queue.Empty:
response = None
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.gateway.deprioritizeComputation(computationId)
def evalApplyTuple(applyTuple, signature = None):
varNames = map(lambda ix: "x_%s" % ix, range(len(applyTuple)))
if signature is None:
evalStringTerms = varNames
else:
evalStringTerms = []
for ix in range(len(signature.terms)):
term = signature.terms[ix]
if term.isNormal():
name = term.asNormal.name
if name is not None:
evalStringTerms.append("%s: x_%s" % (name, ix))
else:
evalStringTerms.append("x_%s" % ix)
elif term.isTupleCall():
evalStringTerms.append("*x_%s" % ix)
evalString = evalStringTerms[0] + "`(" + ",".join(evalStringTerms[1:]) + ")"
try:
return normalComputationResult(
FORA.extractImplValContainer(
FORA.eval(
evalString,
locals = { name: val
for (name, val) in
zip(varNames, applyTuple) },
keepAsForaValue = True
)
)
)
except ForaValue.FORAException as e:
return exceptionComputationResult(
FORA.extractImplValContainer(e.foraVal)
)
def test_futures_7(self):
funString = "fun(f, x) { let res = f(x, *x); res + 1 }"
cfg = self.parseStringToFunction(funString).toCFG(2)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(*x) { size(x) }"))
xImplVal = ForaNative.ImplValContainer((1,2,3));
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, "block_0Let", (fImplval, xImplVal))
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.submittableArgs(0)
cfgWithFutures.slotCompleted(
0, normalComputationResult(ForaNative.ImplValContainer(4))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [1])
cfgWithFutures.slotCompleted(
1, normalComputationResult(ForaNative.ImplValContainer(5))
)
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
cfgWithFutures.continueSimulation()
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(5)
)
def test_gradientBoostedClassificationSamplingSplits_2(self):
score = FORA.eval(
"""
let builder =
math.ensemble.gradientBoosting.GradientBoostedClassifierBuilder(
splitMethod:`sample, nBoosts: 100, maxDepth:1
);
let fit = builder.fit(X, y);
// this answer diverges a bit from scikit, and I'm not exactly sure
// why. however, as noted in the regression tests, we're not
// using exactly the same splitting criteron as scikit
fit.score(X, y)""", {
'X': self.irisX(),
'y': self.irisY()
})
self.assertAlmostEqual(score, 0.8733333333333333)
def disabled_garbage_collection_with_serial_applies(self):
cfg = self.parseStringToFunction(
"fun(f) { f(1) + f(2) + f(3) }").toCFG(1)
nodeValues = (FORA.extractImplValContainer(
FORA.eval("fun(x) { x }")), )
serial = serialSimulation(cfg, nodeValues)
self.assertTrue(serial.getSlots()[0].isGarbageCollected())
self.assertTrue(serial.getSlots()[1].isGarbageCollected())
self.assertFalse(serial.getSlots()[2].isGarbageCollected())
self.seed(10)
for ix in range(10):
randomOrder = randomOrderSimulation(cfg, nodeValues)
self.assertTrue(randomOrder.getSlots()[0].isGarbageCollected())
self.assertTrue(randomOrder.getSlots()[1].isGarbageCollected())
self.assertFalse(randomOrder.getSlots()[2].isGarbageCollected())
def test_tuple_args_1(self):
text = "fun(f, x) { f(*x) }"
cfg = self.parseStringToFunction(text).toCFG(2)
fImplval = FORA.extractImplValContainer(
FORA.eval("let f = fun(x, y, z) { x + y + z }; f")
)
xImplVal = ForaNative.ImplValContainer((1,2,3));
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (fImplval, xImplVal))
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
)
self.assertEqual(
cfgWithFutures.getFinalResult().asResult.result.asResult.result,
ForaNative.ImplValContainer(6)
)
def test_sortALargeVectorWithAdding(self):
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let v = Vector.range(12500000, {_%3.1415926});
if (sorting.isSorted(sorting.sort(v))) 'sorted_2'
}"""
)
)
computationId = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
time.sleep(10)
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
try:
response = self.gateway.finalResponses.get(timeout=360.0)
except Queue.Empty:
response = None
self.gateway.deprioritizeComputation(computationId)
if response is None:
try:
dumpFun = self.dumpSchedulerEventStreams
dumpFun()
except:
logging.warn("Wanted to dump CumulusWorkerEvents, but couldn't");
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == 'sorted_2', response[1].asResult.result.pyval)
def test_resumingAfterCopyDataOutOfPages(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInternalSplitting=False)
text = """
fun() {
let v = Vector.range(1000).paged;
let ix1 = 0
let res = 0
while (true) {
res = res + v[ix1]
ix1 = (ix1 + 1) % size(v)
}
res
}"""
context.placeInEvaluationState(
FORANative.ImplValContainer(
(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)))
context.interruptAfterCycleCount(100000)
context.compute()
paused1 = context.extractPausedComputation()
while not context.isVectorLoad():
context.copyValuesOutOfVectorPages()
vdm.unloadAllPossible()
context.resetInterruptState()
context.interruptAfterCycleCount(100000)
context.compute()
paused2 = context.extractPausedComputation()
self.assertTrue(
len(paused1.asThread.computation.frames) == len(
paused2.asThread.computation.frames))
def copyDataOutOfPagesTest(self, text, cycleCount, expectsToHaveCopies):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.configuration.agressivelyValidateRefcountsAndPageReachability = True
context.configuration.releaseVectorHandlesImmediatelyAfterExecution = False
context.interruptAfterCycleCount(cycleCount)
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
if expectsToHaveCopies:
self.assertTrue(context.copyValuesOutOfVectorPages())
self.assertFalse(context.copyValuesOutOfVectorPages())
else:
self.assertFalse(context.copyValuesOutOfVectorPages())
def test_futures_3(self):
cfg = self.parseStringToFunction("fun(f) { f(1) + f(2) }").toCFG(1)
funImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, "block_0Let", (funImplval,))
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_2Try")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])
cfgWithFutures.submittableArgs(1)
cfgWithFutures.slotCompleted(
1, normalComputationResult(ForaNative.ImplValContainer(3))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.submittableArgs(0)
cfgWithFutures.slotCompleted(
0, normalComputationResult(ForaNative.ImplValContainer(2))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2])
cfgWithFutures.submittableArgs(2)
cfgWithFutures.slotCompleted(
2, normalComputationResult(ForaNative.ImplValContainer(5))
)
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [])
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_2Try")
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(5)
)
def test_switch_nodes_2(self):
text = """fun(x, f) {
let res = f(x);
match (res) with
(1) { -1 }
(2) { -2 }
}"""
cfg = self.parseStringToFunction(text).toCFG(2)
fImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
xImplVal = ForaNative.ImplValContainer(1)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (xImplVal, fImplval))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Let")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.slotCompleted(
0, evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
)
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_1Try")
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_5Try")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [1])
cfgWithFutures.slotCompleted(
1, evalSubmittableArgs(cfgWithFutures.submittableArgs(1))
)
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_5Try")
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(
finalResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(-2)
)
def test_futures_with_exceptions_3(self):
cfg = self.parseStringToFunction("fun(f) { f(1) + f(2) }").toCFG(1)
funImplval = FORA.extractImplValContainer(
FORA.eval("fun(x) { throw 42; }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, "block_0Let", (funImplval, ))
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])
cfgWithFutures.slotCompleted(
1, exceptionComputationResult(ForaNative.ImplValContainer(42)))
self.assertFalse(cfgWithFutures.hasResolvedToSimpleState())
self.assertTrue(cfgWithFutures.mustBailEarly())
cfgWithFutures.slotCompleted(
0, exceptionComputationResult(ForaNative.ImplValContainer(420)))
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(finalResult.asPaused.frame.label, "block_5Throw")
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, finalResult.asPaused.frame.label,
finalResult.asPaused.frame.values)
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
cfgWithFutures.slotCompleted(
0, exceptionComputationResult(ForaNative.ImplValContainer(1337)))
self.assertTrue(cfgWithFutures.hasResolvedToSimpleState())
finalResult = cfgWithFutures.getFinalResult()
self.assertEqual(finalResult.asResult.result.asException.exception,
ForaNative.ImplValContainer(1337))
def test_teardown_during_vector_load(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(dataManager=vdm)
context.evaluate(
FORA.extractImplValContainer(
FORA.eval("fun() { let v = [1,2,3].paged; fun() { v[1] } }")),
FORANative.symbol_Call)
vdm.unloadAllPossible()
pagedVecAccessFun = context.getFinishedResult().asResult.result
context.teardown()
context.evaluate(pagedVecAccessFun, FORANative.symbol_Call)
self.assertFalse(context.isInterrupted())
self.assertTrue(context.isVectorLoad())
context.teardown()
def test_gradientBoostedRegressionAbsoluteLoss(self):
score = FORA.eval(
"""
let builder =
math.ensemble.gradientBoosting.GradientBoostedRegressorBuilder(
splitMethod:`exhaustive, nBoosts: 1, loss: `lad, maxDepth: 2
);
let fit = builder.fit(X,y);
fit.score(X, y)""", {
'X': self.X(),
'y': self.y()
})
# python gives a slightly different answer here: 0.7147891405349398
# however, for gradient boosting, they use a different split criteria
# instead of the most basic mean square error: they use a modification
# due to Friedman which can be found in his original paper on Boosting,
# or in the scikit source.
self.assertAlmostEqual(score, 0.716655282631498)
def test_serialize_during_vector_load(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInternalSplitting=False)
evaluate(
context,
FORA.extractImplValContainer(
FORA.eval("fun(){ datasets.s3('a','b')[0] }")),
FORANative.symbol_Call)
self.assertTrue(context.isVectorLoad())
serialized = context.serialize()
context2 = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInternalSplitting=False)
context2.deserialize(serialized)
self.assertTrue(context2.isVectorLoad())
def submitComputationOnClient(self, clientIndex, expressionText,
**freeVariables):
if isinstance(expressionText, CumulusNative.ComputationDefinition):
computationDefinition = expressionText
else:
varNames = list(freeVariables.keys())
expr = FORA.eval("fun(" + ",".join(varNames) + ") {" +
expressionText + " } ")
computationDefinition = createComputationDefinition(
FORA.extractImplValContainer(expr),
ForaNative.makeSymbol("Call"),
*[freeVariables[v] for v in varNames])
computationId = self.getClient(clientIndex).createComputation(
computationDefinition)
self.getClient(clientIndex).setComputationPriority(
computationId, CumulusNative.ComputationPriority(1))
return computationId
def test_loop_unrolling(self):
text = """fun(f) {
let res = 0;
let ix = 0;
while (true)
{
f(ix);
ix = ix + 1
}
res
}"""
cfg = self.parseStringToFunction(text).toCFG(1)
fImplVal = FORA.extractImplValContainer(FORA.eval("fun(x) { x ** 2 }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (fImplVal,)
)
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
self.assertEqual(len(cfgWithFutures.indicesOfSubmittableFutures()), 2)
cfgWithFutures.slotCompleted(
1, evalSubmittableArgs(cfgWithFutures.submittableArgs(1))
)
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
self.assertEqual(len(cfgWithFutures.indicesOfSubmittableFutures()), 3)
cfgWithFutures.slotCompleted(
3, evalSubmittableArgs(cfgWithFutures.submittableArgs(3))
)
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
self.assertEqual(len(cfgWithFutures.indicesOfSubmittableFutures()), 4)
def test_expensive_brownian(self):
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval("""fun() {
let brownian = fun(x,t) {
if (t == 0)
return sum(0.0, x * 10.0**7.0)
else
{
let (l,r) = cached(brownian(x - 1, t - 1), brownian(x + 1, t - 1));
return sum(0.0, x * 10.0**7.0) + l + r
}
}
brownian(0, 5)
}"""))
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(expr,
ForaNative.makeSymbol("Call")))
self.assertTrue(response[1].isResult())
def test_mixedVecOfNothingAndFloat(self):
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let v = Vector.range(1000000, fun(x) { if (x%5 == 0) nothing else x });
sum(0,10, fun(ix) { v.apply(fun (nothing) { nothing } (elt) { elt + ix }).sum() })
}"""
)
)
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == 4000036000000, response[1].asResult.result.pyval)
def test_cannotSplitThingsWithMutables(self):
text = """fun() {
let s = fun(a,b,f) {
if (b <= a)
return nothing
if (b <= a + 1)
return f(a)
let mid = Int64((a+b)/2)
return s(a,mid,f) + s(mid,b,f)
}
let m = MutableVector.create(10,10);
s(0,1000,fun(x){m[0] * x})
}"""
funImplval = FORA.extractImplValContainer(FORA.eval(text))
i = 0
while (i < 30000):
i += 1000
try:
pausedComputation = callAndExtractPausedCompuationAfterSteps(
funImplval, i)
except NotInterruptedException as e:
break
allAreCST = True
for val in pausedComputation.frames[0].values:
if not val.isCST():
allAreCST = False
break
if not allAreCST:
splitComputation = ForaNative.splitPausedComputation(
pausedComputation)
self.assertTrue(not splitComputation, splitComputation)
def test_bigSumWithAdding(self):
self.desirePublisher.desireNumberOfWorkers(1, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() { if (sum(0, 10**11) > 0) 'big_enough' }"""
)
)
computationId = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
time.sleep(5)
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
try:
response = self.gateway.finalResponses.get(timeout=240.0)
except Queue.Empty:
response = None
if response is None:
try:
dumpFun = self.dumpSchedulerEventStreams
dumpFun()
except:
logging.warn("Wanted to dump CumulusWorkerEvents, but couldn't");
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == 'big_enough', response[1].asResult.result.pyval)
self.gateway.deprioritizeComputation(computationId)
def test_extractPausedComputation(self):
text = """fun() {
let x = 0;
while (x < 100000)
x = x + 1
x
}"""
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.interruptAfterCycleCount(1010)
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
computation = context.extractPausedComputation()
self.assertEqual(len(computation.frames), 2)
self.assertEqual(computation.frames[1].values[0].pyval, 335)
context2 = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context2.resumePausedComputation(computation)
context2.resume()
self.assertEqual(context2.getFinishedResult().asResult.result.pyval,
100000)
context.teardown()
context2.teardown()
def test_createInstance_2(self):
classIvc = FORA.eval("""let C = class {
member y;
member x;
};
C""").implVal_
x = 1
y = 2
res = ForaNative.simulateApply(
ForaNative.ImplValContainer(
(classIvc, makeSymbolIvc("CreateInstance"),
ForaNative.ImplValContainer(y),
ForaNative.ImplValContainer(x))))
self.assertIsNotNone(res)
computed_x = ForaNative.simulateApply(
ForaNative.ImplValContainer(
(res, self.Symbol_member, makeSymbolIvc("x"))))
self.assertEqual(computed_x, ForaNative.ImplValContainer(x))
def test_serialize_while_holding_interior_vector(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.evaluate(
FORA.extractImplValContainer(
FORA.eval("""
fun() {
let v = [[1].paged].paged;
let v2 = v[0]
`TriggerInterruptForTesting()
1+2+3+v+v2
}""")), FORANative.symbol_Call)
self.assertTrue(context.isInterrupted())
serialized = context.serialize()
context = None
def test_splitComputation_2(self):
text = """fun() {
let s = fun(a,b,f) {
if (b <= a)
return nothing
if (b <= a + 1)
return f(a)
let mid = Int64((a+b)/2)
return s(a,mid,f) + s(mid,b,f)
}
s(0,1000,fun(x){x})
}"""
funImplval = FORA.extractImplValContainer(FORA.eval(text))
pausedComputation = callAndExtractPausedCompuationAfterSteps(
funImplval, 35)
splitComputation = ForaNative.splitPausedComputation(pausedComputation)
self.assertIsNotNone(splitComputation)
unsplitValue = finishPausedComputation(pausedComputation)
applyComputationVal = finishPausedComputation(
splitComputation.applyComputation)
splitComputationVal = finishPausedComputation(
splitComputation.splitComputation)
resumedComputation = ForaNative.joinSplitPausedComputation(
splitComputation, applyComputationVal, splitComputationVal)
finalSplitVal = finishPausedComputation(resumedComputation)
self.assertEqual(unsplitValue, finalSplitVal)
def test_manyDuplicateCachecallsAndAdding(self):
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval("""fun() {
let sumf = fun(a,b) {
if (a + 1 >= b) {
return [a + cached(sum(0,10**11))[0]]
}
let mid = (a+b)/2
return sumf(a,mid) + sumf(mid,b)
}
sumf(0,20)
}"""))
computationId = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(expr,
ForaNative.makeSymbol("Call")))
time.sleep(5)
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
try:
response = self.gateway.finalResponses.get(timeout=240.0)
except Queue.Empty:
response = None
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.gateway.deprioritizeComputation(computationId)
def test_futures_1(self):
cfg = self.parseStringToFunction("fun(f) { f(1) + f(2) }").toCFG(1)
funImplval = FORA.extractImplValContainer(FORA.eval("fun(x) { x + 1 }"))
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, "block_0Let", (funImplval,))
self.assertEqual(cfgWithFutures.currentLabel(), "block_0Let")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
asSubmittable = cfgWithFutures.submittableArgs(0)
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_1Let")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])
self.assertIsNotNone(cfgWithFutures.submittableArgs(1))
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.currentLabel(), "block_2Try")
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0, 1])
self.assertIsNone(cfgWithFutures.submittableArgs(2))
self.assertIsNone(cfgWithFutures.submittableArgs(3))
def test_vectorApplyWithAdding(self):
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let v = [1,2,3,4].paged;
let res = 0
sum(0,20000000000, fun(x) { v[x%4] })
'test_vectorApplyWithAdding'
}"""
)
)
computationId = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
try:
response = self.gateway.finalResponses.get(timeout=5.0)
except Queue.Empty:
response = None
self.assertTrue(response is None, response)
self.desirePublisher.desireNumberOfWorkers(3, blocking=True)
response = self.gateway.finalResponses.get(timeout=240.0)
self.gateway.deprioritizeComputation(computationId)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == 'test_vectorApplyWithAdding', response[1].asResult.result.pyval)
def test_tupleExpand_nodes_1(self):
text = "fun(h) { let (f, g) = h(); (f(), g()) }"
cfg = self.parseStringToFunction(text).toCFG(1)
hImplVal = FORA.extractImplValContainer(
FORA.eval(
"fun() { let f = fun() { 1 }; let g = fun() { 2 }; (f, g) }"
)
)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, None, (hImplVal,))
res = evalSubmittableArgs(cfgWithFutures.submittableArgs(0))
cfgWithFutures.slotCompleted(
0, res
)
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
submittableArgs = cfgWithFutures.submittableArgs(1)
cfgWithFutures.slotCompleted(
1, evalSubmittableArgs(submittableArgs)
)
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2])
# this verifies that parallelism is possible for `text`
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [2, 3])
"""
def test_regressionTrees_5(self):
res = FORA.eval(
"""
let y_median = math.stats.median(y.getColumn(0));
let y_medians = dataframe.Series(Vector.uniform(size(X), y_median));
let neg_gradient = (y.getColumn(0) - y_medians) ~~ math.sign;
neg_gradient = dataframe.DataFrame([neg_gradient])
let treeBuilder =
math.tree.RegressionTree.RegressionTreeBuilder(
maxDepth:2, minSamplesSplit:2,
splitMethod:`exhaustive
);
let tree = treeBuilder.fit(X, neg_gradient)
// checked against scikit
assertions.assertClose(tree.score(X, y), 0.53214422106820458)
""", {
'X': self.X(),
'y': self.y()
})
self.assertTrue(res)
def test_regressionTrees_1(self):
res = FORA.eval(
"""
let regressionTree =
math.tree.RegressionTree.buildTree(
df, yDim: 2, maxDepth: 5, minSamplesSplit: 50
);
let expectedValues =
[-0.953882430908236, 99.7799382797797, -0.953882430908236,
-0.953882430908236, -1.85651176009063, -1.85651176009063,
-0.731458273696544, -0.953882430908236, -0.731458273696544,
-0.731458273696544];
let computedValues =
Vector.range(size(expectedValues)) ~~ {
regressionTree.predict(df("X", "Y")[_])
};
assertions.assertAllClose(expectedValues, computedValues);
computedValues =
Vector.range(size(expectedValues)) ~~ {
regressionTree.predict(df[_])
};
assertions.assertAllClose(expectedValues, computedValues);
let predictedValues = regressionTree.predict(df);
let meanSquareError =
math.stats.mean((predictedValues - df.getColumn("Z")) ** 2);
assertions.assertClose(meanSquareError, 0.986815325486888)""",
{'df': self.generateDataset(1000, 42)})
self.assertTrue(res)
def test_let_bindings_2(self):
locals = {}
FORA.eval("let x = 10; x = 20", locals)
self.assertEqual(locals, {'x': 20})
def test_ForaExceptions_1(self):
with self.assertRaises(ForaValue.FORAException):
FORA.eval("1 / 0")
