def test_cfgSplitting_5(self):
funString = "fun(f) { (f(1) + f(2)) + f(3) }"
cfg = self.parseStringToFunction(funString).toCFG(1)
steps = ForaNative.extractApplyStepsFromControlFlowGraph(cfg, None)
splits = ForaNative.splitControlFlowGraph(cfg, "block_0Let")
self.assertTrue(splits is not None)
def test_futures_tupleCallInContinuation_2(self):
text = "fun(x) { x = x + x; size((x, *x)) }"
cfg = self.parseStringToFunction(text).toCFG(1)
x = 1
xImplVal = ForaNative.ImplValContainer(x)
cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures(
cfg, "block_1Let", (xImplVal,))
cfgWithFutures.continueSimulation()
cfgWithFutures.continueSimulation()
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0])
x = x + x
cfgWithFutures.slotCompleted(
0, normalComputationResult(
ForaNative.ImplValContainer(x)
)
)
self.assertFalse(cfgWithFutures.hasResolvedToSimpleState())
self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [1])
self.assertComponentwiseEqual(
cfgWithFutures.submittableArgs(1).args.values,
[ForaNative.makeSymbol("size"), ForaNative.makeSymbol("Call"),
ForaNative.ImplValContainer((x, x))]
)
def test_extractPausedComputationDuringVectorLoad(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
)
pagedVec = context.getFinishedResult().asResult.result
context.placeInEvaluationState(
FORANative.ImplValContainer(
(pagedVec,
FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
FORANative.ImplValContainer(0))
)
)
vdm.unloadAllPossible()
context.resume()
self.assertTrue(context.isVectorLoad())
computation = context.extractPausedComputation()
self.assertEqual(len(computation.frames),1)
def test_joas(self):
#"test assertions about JOAs of particular axioms"
for item in self.vals_to_test:
jovt = self.getJOVT(item[0])
if len(item) == 4:
assert item[3] == "hasSideEffects", "illegal third argument to JOA"
expected_joa = FORANative.JudgmentOnAction(
FORANative.parseStringToJOR(item[1]),
FORANative.parseStringToJOR(item[2]),
True if len(item) == 4 and item[3] == "hasSideEffects" else False
)
computed_joa = self.joa(jovt)
if computed_joa:
self.assertTrue(
expected_joa.resultPart() == computed_joa.resultPart(),
"for JOVT %s should have had JOA resultPart: %s, but had: %s" \
% (jovt, expected_joa.resultPart(), computed_joa.resultPart())
)
#be a little more relaxed on the throwParts
self.assertTrue(
expected_joa.throwPart().covers(computed_joa.throwPart()),
"for JOVT %s: expected JOA throwPart %s does not cover computed JOA %s" \
% (jovt, expected_joa.throwPart(), computed_joa.throwPart())
)
if len(expected_joa.throwPart()) > 0:
self.assertTrue(
len(computed_joa.throwPart()) > 0,
"for JOVT %s: expected JOA %s throws, but computed JOA %s does not" \
%(jovt, expected_joa, computed_joa)
)
def convertClassInstanceDescription(self, objectId,
classInstanceDescription):
classMemberNameToImplVal = {
classMemberName: self.convertedValues[memberId]
for classMemberName, memberId in classInstanceDescription.
classMemberNameToClassMemberId.iteritems()
}
classImplVal = self.convertedValues[classInstanceDescription.classId]
if classImplVal.isSymbol():
self.convertedValues[objectId] = classImplVal
return
memberNames = tuple(
sorted(name for name in classMemberNameToImplVal.iterkeys()))
memberValues = tuple(classMemberNameToImplVal[name]
for name in memberNames)
convertedValueOrNone = ForaNative.simulateApply(
ForaNative.ImplValContainer(
(classImplVal, Symbol_CreateInstance,
ForaNative.CreateNamedTuple(memberValues, memberNames))))
if convertedValueOrNone is None:
raise pyfora.PythonToForaConversionError(
("An internal error occurred: " +
"function stage 1 simulation unexpectedly returned None"))
self.convertedValues[objectId] = convertedValueOrNone
def test_createInstance_2(self):
classIvc = FORA.eval(
"""let C = class {
member y;
member x;
};
C"""
).implVal_
x = 1
y = 2
res = ForaNative.simulateApply(
ForaNative.ImplValContainer(
(
classIvc,
makeSymbolIvc("CreateInstance"),
ForaNative.ImplValContainer(y),
ForaNative.ImplValContainer(x),
)
)
)
self.assertIsNotNone(res)
computed_x = ForaNative.simulateApply(
ForaNative.ImplValContainer((res, self.Symbol_member, makeSymbolIvc("x")))
)
self.assertEqual(computed_x, ForaNative.ImplValContainer(x))
def test_TypedFora_CreateTuple(self):
layoutStyles = [
TypedFora.RefcountStyle.Pooled(),
TypedFora.RefcountStyle.AsValueUnowned(),
TypedFora.RefcountStyle.AsValueOwned()
]
jovs = [
ForaNative.parseStringToJOV("{String}"),
ForaNative.parseStringToJOV("'a string'"),
ForaNative.parseStringToJOV("*")
]
aVal = ForaNative.ImplValContainer("a string")
allLayouts = []
for ls1 in layoutStyles:
for jov1 in jovs:
allLayouts.append(TypedFora.Type(jov1, ls1))
for t1 in allLayouts:
for t2 in allLayouts:
callable = self.generateTupleCallable([t1, t2], [False, False])
def validator(result):
return result == ForaNative.ImplValContainer((aVal, aVal))
self.runSimpleEvaluation(callable, [aVal, aVal], validator)
def test_TypedFora_Layout_UsingCompiler(self):
layoutStyles = [
TypedFora.RefcountStyle.Pooled(),
TypedFora.RefcountStyle.AsValueUnowned(),
TypedFora.RefcountStyle.AsValueOwned()
]
jovs = [
ForaNative.parseStringToJOV("{String}"),
#a small union
ForaNative.parseStringToJOV("{Union([{String}, {Int64}])}"),
#a much bigger union
ForaNative.parseStringToJOV("{Union([{String}, {Int64}, {Float64}, nothing, ({Float64})])}"),
ForaNative.parseStringToJOV("'%s'" % aBigString),
ForaNative.parseStringToJOV("*")
]
for ls1 in layoutStyles:
for jov1 in jovs:
for ls2 in layoutStyles:
for jov2 in jovs:
if not (ls1.isAsValueOwned() and ls2.isAsValueUnowned()):
t1 = TypedFora.Type(jov1, ls1)
t2 = TypedFora.Type(jov2, ls2)
self.runSimpleEvaluation(t1, t2)
def copyDataOutOfPagesTest(self, text, cycleCount, expectsToHaveCopies):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm,
allowInterpreterTracing=False,
allowInternalSplitting=False)
context.configuration.agressivelyValidateRefcountsAndPageReachability = True
context.configuration.releaseVectorHandlesImmediatelyAfterExecution = False
context.placeInEvaluationState(
FORANative.ImplValContainer(
(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)))
context.interruptAfterCycleCount(cycleCount)
context.compute()
if expectsToHaveCopies:
self.assertTrue(context.copyValuesOutOfVectorPages())
self.assertFalse(context.copyValuesOutOfVectorPages())
else:
self.assertFalse(context.copyValuesOutOfVectorPages())
def createWorkersAndClients(workerCount,
clientCount,
viewFactory=None,
memoryLimitMb=100,
threadCount=2,
callbackSchedulerToUse=None):
if callbackSchedulerToUse is None:
callbackSchedulerToUse = CallbackScheduler.singletonForTesting()
if viewFactory is None:
viewFactory = createInMemorySharedStateViewFactory(
callbackSchedulerToUse)
workersVdmsAndEventHandlers = [
createWorker(machineId(ix),
viewFactory,
memoryLimitMb=memoryLimitMb,
threadCount=threadCount,
callbackSchedulerToUse=callbackSchedulerToUse)
for ix in range(workerCount)
]
clientsAndVdms = [
createClient(clientId(ix),
callbackSchedulerToUse=callbackSchedulerToUse)
for ix in range(clientCount)
]
for ix1 in range(len(workersVdmsAndEventHandlers)):
workersVdmsAndEventHandlers[ix1][0].startComputations()
for ix1 in range(len(workersVdmsAndEventHandlers) - 1):
for ix2 in range(ix1 + 1, len(workersVdmsAndEventHandlers)):
worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel(
callbackSchedulerToUse)
worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel(
callbackSchedulerToUse)
workersVdmsAndEventHandlers[ix1][0].addMachine(
machineId(ix2), [worker1Channel1, worker1Channel2],
ForaNative.ImplValContainer(), callbackSchedulerToUse)
workersVdmsAndEventHandlers[ix2][0].addMachine(
machineId(ix1), [worker2Channel1, worker2Channel2],
ForaNative.ImplValContainer(), callbackSchedulerToUse)
for ix1 in range(len(workersVdmsAndEventHandlers)):
for ix2 in range(len(clientsAndVdms)):
workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel(
callbackSchedulerToUse)
workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel(
callbackSchedulerToUse)
workersVdmsAndEventHandlers[ix1][0].addCumulusClient(
clientId(ix2), [workerChannel1, workerChannel2],
ForaNative.ImplValContainer(), callbackSchedulerToUse)
clientsAndVdms[ix2][0].addMachine(machineId(ix1),
[clientChannel1, clientChannel2],
ForaNative.ImplValContainer(),
callbackSchedulerToUse)
return workersVdmsAndEventHandlers, clientsAndVdms, viewFactory
def test_resumePausedComputationWithResult(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm,
allowInterpreterTracing=False,
allowInternalSplitting=False)
text = """
let f = fun(v, ix) {
if (ix > 0)
{
let (v2,res) = f(v,ix-1);
return (v2, res + v2[0])
}
`TriggerInterruptForTesting()
return (v, 0)
};
f([1], 10)
"""
evaluate(
context,
FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call")))
assert context.isInterrupted()
pausedComp = context.extractPausedComputation()
framesToUse = pausedComp.asThread.computation.frames[0:5]
pausedComp2 = FORANative.PausedComputationTree(
FORANative.PausedComputation(
framesToUse,
FORA.extractImplValContainer(
FORA.eval("([2], 0)", keepAsForaValue=True)), False))
context.resumePausedComputation(pausedComp2)
context.copyValuesOutOfVectorPages()
context.pageLargeVectorHandles(0)
context.resetInterruptState()
context.compute()
self.assertTrue(context.isFinished())
result = context.getFinishedResult()
self.assertTrue(result.asResult.result[1].pyval == 6)
def getCacheRequestComputationResult(cacheCallElement):
bucketname = cacheCallElement[1].pyvalOrNone
keyname = cacheCallElement[2].pyvalOrNone
if not isinstance(bucketname, str) or not isinstance(keyname, str):
return ForaNative.ComputationResult.Exception(
ForaNative.ImplValContainer("Badly formed S3 dataset request: %s" % cacheCallElement)
)
s3Interface = getCurrentS3Interface()
if s3Interface.keyExists(bucketname, keyname):
keysAndSizesMatching = [(keyname, s3Interface.getKeySize(bucketname, keyname))]
else:
keysAndSizesMatching = s3Interface.listKeysWithPrefix(bucketname, keyname + "_")
indicesKeysAndSizes = []
for key, size, mtime in keysAndSizesMatching:
try:
index = int(key[len(keyname)+1:])
indicesKeysAndSizes.append((index, key, size))
except ValueError:
pass
keysAndSizesMatching = [(key, size) for _, key, size in sorted(indicesKeysAndSizes)]
if not keysAndSizesMatching:
return ForaNative.ComputationResult.Exception(
ForaNative.ImplValContainer("No keys matching %s/%s using %s" % (
bucketname,
keyname,
s3Interface
))
)
wholeVectorIVC = ForaNative.getEmptyVector()
for key, size in keysAndSizesMatching:
CHUNK_SIZE = 10 * 1024 * 1024
chunks = getAppropriateChunksForSize(size, CHUNK_SIZE)
for lowIndex, highIndex in chunks:
externalDatasetDesc = ForaNative.ExternalDatasetDescriptor.S3Dataset(
bucketname,
key,
lowIndex,
highIndex
)
vectorDataId = ForaNative.VectorDataID.External(externalDatasetDesc)
vectorIVC = ForaNative.createFORAFreeBinaryVector(vectorDataId, highIndex - lowIndex)
wholeVectorIVC = ForaNative.concatenateVectors(wholeVectorIVC, vectorIVC)
return ForaNative.ComputationResult.Result(wholeVectorIVC, ForaNative.ImplValContainer())
def test_serialResults_5(self):
text = "fun(x) { x = x + x; size((x, *x)) }"
cfg = self.parseStringToFunction(text).toCFG(1)
xImplVal = ForaNative.ImplValContainer(1)
serialResult = self.serialResult(cfg, (xImplVal, ), "block_1Let")
self.assertEqual(serialResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(2))
def test_cfgSplitting_1(self):
cfg1 = self.parseStringToFunction("fun(f) { f(1) + f(1+3) }").toCFG(1)
steps = ForaNative.extractApplyStepsFromControlFlowGraph(cfg1, "block_0Let")
self.assertEqual(len(steps), 4)
splits = ForaNative.splitControlFlowGraph(cfg1, "block_0Let")
self.assertTrue(splits is not None)
def getCacheRequestComputationResult(cacheCallElement):
bucketname = cacheCallElement[1].pyvalOrNone
keyname = cacheCallElement[2].pyvalOrNone
if not isinstance(bucketname, str) or not isinstance(keyname, str):
return ForaNative.ComputationResult.Exception(
ForaNative.ImplValContainer("Badly formed S3 dataset request: %s" %
cacheCallElement))
s3Interface = getCurrentS3Interface()
if s3Interface.keyExists(bucketname, keyname):
keysAndSizesMatching = [(keyname,
s3Interface.getKeySize(bucketname, keyname))]
else:
keysAndSizesMatching = s3Interface.listKeysWithPrefix(
bucketname, keyname + "_")
indicesKeysAndSizes = []
for key, size, mtime in keysAndSizesMatching:
try:
index = int(key[len(keyname) + 1:])
indicesKeysAndSizes.append((index, key, size))
except ValueError:
pass
keysAndSizesMatching = [
(key, size) for _, key, size in sorted(indicesKeysAndSizes)
]
if not keysAndSizesMatching:
return ForaNative.ComputationResult.Exception(
ForaNative.ImplValContainer("No keys matching %s/%s using %s" %
(bucketname, keyname, s3Interface)))
wholeVectorIVC = ForaNative.getEmptyVector()
for key, size in keysAndSizesMatching:
CHUNK_SIZE = 10 * 1024 * 1024
chunks = getAppropriateChunksForSize(size, CHUNK_SIZE)
for lowIndex, highIndex in chunks:
externalDatasetDesc = ForaNative.ExternalDatasetDescriptor.S3Dataset(
bucketname, key, lowIndex, highIndex)
vectorDataId = ForaNative.VectorDataID.External(
externalDatasetDesc)
vectorIVC = ForaNative.createFORAFreeBinaryVector(
vectorDataId, highIndex - lowIndex)
wholeVectorIVC = ForaNative.concatenateVectors(
wholeVectorIVC, vectorIVC)
return ForaNative.ComputationResult.Result(wholeVectorIVC)
def testMatchStructureCantMatchTooSmallTuple(self):
runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()
jov = FORANative.parseStringToJOV("((1,2,3),`StructureMatch,((nothing, false), (nothing, false), (nothing, false), (nothing, false)))")
axiom = axioms.getAxiomByJOVT(runtime.getTypedForaCompiler(), jov.asTuple.jov)
self.assertTrue(list(axiom.asNative.resultSignature.resultPart().vals) == [FORANative.parseStringToJOV("nothing")])
def evalSubmittableArgs(submittableArgs):
def evalApplyTuple(applyTuple, signature=None):
varNames = map(lambda ix: "x_%s" % ix, range(len(applyTuple)))
if signature is None:
evalStringTerms = varNames
else:
evalStringTerms = []
for ix in range(len(signature.terms)):
term = signature.terms[ix]
if term.isNormal():
name = term.asNormal.name
if name is not None:
evalStringTerms.append("%s: x_%s" % (name, ix))
else:
evalStringTerms.append("x_%s" % ix)
elif term.isTupleCall():
evalStringTerms.append("*x_%s" % ix)
evalString = evalStringTerms[0] + "`(" + ",".join(
evalStringTerms[1:]) + ")"
try:
return normalComputationResult(
FORA.extractImplValContainer(
FORA.eval(evalString,
locals={
name: val
for (name, val) in zip(varNames, applyTuple)
},
keepAsForaValue=True)))
except ForaValue.FORAException as e:
return exceptionComputationResult(
FORA.extractImplValContainer(e.foraVal))
if submittableArgs.isApply():
return evalApplyTuple(submittableArgs.args.values,
submittableArgs.args.signature)
else: # submittableArgs.isCached()
tr = ()
# this corresponds to the logic for Cached nodes in
# CFGWithFutures::asSubmittable
assert len(submittableArgs.args.values) == 1
for cachedPair in submittableArgs.args.values[0]:
applyArgs = [cachedPair[0], ForaNative.makeSymbol("Call")]
applyArgs.extend(cachedPair[1])
res = evalApplyTuple(applyArgs)
if res.isException():
return res
tr = tr + (res.asResult.result, )
return normalComputationResult(ForaNative.ImplValContainer(tr))
def test_vectorOfPagedVectorApplyWithDropping(self):
self.desirePublisher.desireNumberOfWorkers(3, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let v =
Vector.range(20).apply(fun(ix) {
Vector.range(1250000.0+ix).paged
}).paged
let lookup = fun(ix) { v[ix] }
Vector.range(100).apply(fun(ix) { sum(0, 10**8); cached(lookup(ix)) })
v
}"""
)
)
computation1 = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
try:
response = self.gateway.finalResponses.get(timeout=60.0)
except Queue.Empty:
response = None
self.assertTrue(response is not None, response)
self.gateway.deprioritizeComputation(computation1)
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr2 = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let res = 0
for ix in sequence(10)
res = res + Vector.range(12500000+ix).sum()
return res
}"""
)
)
computation2 = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(
expr2,
ForaNative.makeSymbol("Call")
)
)
try:
response = self.gateway.finalResponses.get(timeout=60.0)
except Queue.Empty:
response = None
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.gateway.deprioritizeComputation(computation2)
def test_cfgSplitting_1(self):
cfg1 = self.parseStringToFunction("fun(f) { f(1) + f(1+3) }").toCFG(1)
steps = ForaNative.extractApplyStepsFromControlFlowGraph(
cfg1, "block_0Let")
self.assertEqual(len(steps), 4)
splits = ForaNative.splitControlFlowGraph(cfg1, "block_0Let")
self.assertTrue(splits is not None)
def test_simulateGetItem(self):
n = 42
ivc = FORA.eval("object { m: %s }" % n).implVal_
res = ForaNative.simulateApply(
ForaNative.ImplValContainer(
(ivc, self.Symbol_member, makeSymbolIvc("m"))))
self.assertEqual(res, ForaNative.ImplValContainer(n))
def test_TypedFora_TupleGetItem(self):
tupJovs = [
ForaNative.parseStringToJOV("({String}, {String})"),
ForaNative.parseStringToJOV("({String}, *)"),
ForaNative.parseStringToJOV("({String}, 'a string 2')"),
ForaNative.parseStringToJOV("('a string', 'a string 2')"),
ForaNative.parseStringToJOV("('a string', *)"),
ForaNative.parseStringToJOV("('a string', {String})"),
ForaNative.parseStringToJOV("(*, 'a string 2')"),
ForaNative.parseStringToJOV("(*, *)"),
ForaNative.parseStringToJOV("(*, {String})")
]
layoutStyles = [
TypedFora.RefcountStyle.Pooled(),
TypedFora.RefcountStyle.AsValueUnowned(),
TypedFora.RefcountStyle.AsValueOwned()
]
instance = ForaNative.ImplValContainer( ("a string", "a string 2") )
for tupJov in tupJovs:
for refcountStyle in layoutStyles:
for index in range(2):
type = TypedFora.Type(tupJov, refcountStyle)
callable = self.generateTupleGetitemCallable(type, index)
def validator(result):
return result == instance[index]
self.runSimpleEvaluation(callable, [instance], validator)
def wireWorkerToClient_(self, ix1, ix2):
workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
self.workersVdmsAndEventHandlers[ix1][0].addCumulusClient(
clientId(ix2), [workerChannel1, workerChannel2],
ForaNative.ImplValContainer(), self.callbackScheduler)
self.clientsAndVdms[ix2][0].addMachine(
self.machineIds[ix1], [clientChannel1, clientChannel2],
ForaNative.ImplValContainer(), self.callbackScheduler)
def test_covers(self):
vdm = FORANative.VectorDataManager(
callbackScheduler,
Setup.config().maxPageSizeInBytes
)
context = ExecutionContext.ExecutionContext(dataManager = vdm)
randomJOVGenerator = FORANative.RandomJOVGenerator(0, context)
self.treeElementCoveringTest(coverageTree, [], [], randomJOVGenerator)
def _testJOVTextSet(self, jovStrings, jovMappings):
jovts = [ForaNative.parseStringToJOVT(x) for x in jovStrings]
tree = ForaNative.JudgmentOnValueTree(jovts)
for jmtString, targetIx in jovMappings.iteritems():
testJovt = ForaNative.parseStringToJOVT(jmtString)
searchedIx = tree.searchForJOVT(testJovt)
self.assertEqual(
targetIx, searchedIx,
"JOV Tree:\n%s\nError: %s mapped to %s instead of %s" %
(str(tree), str(testJovt), searchedIx, targetIx))
def runSimpleEvaluation(self, inputType, outputType):
mainRuntime = Runtime.getMainRuntime()
foraCompiler = mainRuntime.getTypedForaCompiler()
while foraCompiler.anyCompilingOrPending():
time.sleep(.01)
aParticularStringValue = ForaNative.ImplValContainer(aBigString)
callable = self.generateSimpleCallable(inputType, outputType)
jumpTarget = foraCompiler.compile(callable)
import gc
gc.collect()
for passIndex in range(PASS_COUNT):
#type values are memoized, so we can't assume that the value has a refcount
# of exactly one
totalStringCount = ForaNative.totalStringCount()
totalImplvalCount = ForaNative.totalImplvalCount()
anExecutionContext = ExecutionContext.ExecutionContext(
dataManager = VectorDataManager.constructVDM(self.callbackScheduler)
)
anExecutionContext.evaluateFunctionPointer(jumpTarget, aParticularStringValue)
self.assertTrue(anExecutionContext.isFinished())
res = anExecutionContext.getFinishedResult()
self.assertTrue(not res.isException())
self.assertEqual(res.asResult.result, aParticularStringValue)
anExecutionContext.teardown()
res = None
#verify final refcounts
self.assertEqual(
aParticularStringValue.getStringObjectRefcount(),
1,
"refcounts weren't maintained in %s->%s. %s != 1" % (
inputType,
outputType,
aParticularStringValue.getStringObjectRefcount()
)
)
self.assertEqual(
(totalStringCount, totalImplvalCount),
(ForaNative.totalStringCount(), ForaNative.totalImplvalCount()),
"refcounts weren't maintained in " + str(inputType) + "->" + str(outputType)
)
def parseStringToPythonAst(text):
lineOffsets = computeLineOffsets(text)
try:
if text not in astCache_:
astCache_[text] = convertPythonAstToForaPythonAst(
ast.parse(text), lineOffsets)
return astCache_[text]
except SyntaxError as e:
return ForaNative.PythonParseError(e.msg, e.filename, e.lineno,
e.offset, e.text)
except TypeError as e:
return ForaNative.PythonParseError(str(e.message))
def test_serialResults_3(self):
funString = "fun(f, x) { let res = f((x, *x)); res + 1 }"
cfg = self.parseStringToFunction(funString).toCFG(2)
fImplval = FORA.extractImplValContainer(
FORA.eval("fun(x) { size(x) }"))
xImplVal = ForaNative.ImplValContainer((1, 2, 3))
nodeValues = (fImplval, xImplVal)
serialResult = self.serialResult(cfg, nodeValues, "block_5Let")
self.assertEqual(serialResult.asResult.result.asResult.result,
ForaNative.ImplValContainer(5))
def calculateTime(count):
t0 = time.time()
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr, ForaNative.makeSymbol("Call"),
ForaNative.ImplValContainer(count)))
self.assertTrue(response[1].isResult())
stats = response[2]
return (stats.timeSpentInInterpreter, stats.timeSpentInCompiler,
time.time() - t0)
def test_resolveAxiomDirectly_smallStrings(self):
instance = ForaNative.ImplValContainer(
("s1", ForaNative.makeSymbol("Operator"),
ForaNative.makeSymbol("+"), "s2"))
jov = ForaNative.implValToJOV(instance)
joa = self.axioms.resolveAxiomDirectly(self.compiler, jov.getTuple())
self.assertEqual(len(joa.throwPart()), 0)
self.assertEqual(len(joa.resultPart()), 1)
result = joa.resultPart()[0]
self.assertEqual(result, ForaNative.parseStringToJOV('"s1s2"'))
def instantiateException(self, exceptionTypeName, exceptionArgsAsPyTuple):
"""Return an IVC representing an exception of type given by name 'exceptionTypeName'.
Returns None if 'exceptionTypeName' isn't a singleton.
"""
exceptionTypeInstance = self.convertSingletonByName(exceptionTypeName)
if exceptionTypeInstance is None:
return None
args = (self.pyExceptionClass, ForaNative.makeSymbol("CreateInstance"),
exceptionTypeInstance, exceptionArgsAsPyTuple)
return ForaNative.simulateApply(ForaNative.ImplValContainer(args))
def test_resolveAxiomDirectly_VeryLongComputation(self):
vectorIVC = FORA.extractImplValContainer(FORA.eval("[]"))
jov = ForaNative.parseStringToJOV(("({Vector([])}, `append, 2)"))
joa = self.axioms.resolveAxiomDirectly(self.compiler, jov.getTuple())
self.assertEqual(len(joa.throwPart()),0)
self.assertEqual(len(joa.resultPart()),1)
result = joa.resultPart()[0]
self.assertEqual(result, ForaNative.parseStringToJOV("{Vector([{Int64}])}"))
def test_resolveAxiomDirectly_smallStrings(self):
instance = ForaNative.ImplValContainer(
("s1", ForaNative.makeSymbol("Operator"), ForaNative.makeSymbol("+"), "s2")
)
jov = ForaNative.implValToJOV(instance)
joa = self.axioms.resolveAxiomDirectly(self.compiler, jov.getTuple())
self.assertEqual(len(joa.throwPart()),0)
self.assertEqual(len(joa.resultPart()),1)
result = joa.resultPart()[0]
self.assertEqual(result, ForaNative.parseStringToJOV('"s1s2"'))
def runSimpleEvaluation(self, inputType, outputType):
mainRuntime = Runtime.getMainRuntime()
foraCompiler = mainRuntime.getTypedForaCompiler()
while foraCompiler.anyCompilingOrPending():
time.sleep(.01)
aParticularStringValue = ForaNative.ImplValContainer(aBigString)
callable = self.generateSimpleCallable(inputType, outputType)
jumpTarget = foraCompiler.compile(callable)
import gc
gc.collect()
for passIndex in range(PASS_COUNT):
#type values are memoized, so we can't assume that the value has a refcount
# of exactly one
totalStringCount = ForaNative.totalStringCount()
totalImplvalCount = ForaNative.totalImplvalCount()
anExecutionContext = ExecutionContext.ExecutionContext(
dataManager=VectorDataManager.constructVDM(
self.callbackScheduler))
anExecutionContext.evaluateFunctionPointer(jumpTarget,
aParticularStringValue)
self.assertTrue(anExecutionContext.isFinished())
res = anExecutionContext.getFinishedResult()
self.assertTrue(not res.isException())
self.assertEqual(res.asResult.result, aParticularStringValue)
anExecutionContext.teardown()
res = None
#verify final refcounts
self.assertEqual(
aParticularStringValue.getStringObjectRefcount(), 1,
"refcounts weren't maintained in %s->%s. %s != 1" %
(inputType, outputType,
aParticularStringValue.getStringObjectRefcount()))
self.assertEqual((totalStringCount, totalImplvalCount),
(ForaNative.totalStringCount(),
ForaNative.totalImplvalCount()),
"refcounts weren't maintained in " +
str(inputType) + "->" + str(outputType))
def _testJOVTextSet(self, jovStrings, jovMappings):
jovts = [ForaNative.parseStringToJOVT(x) for x in jovStrings]
tree = ForaNative.JudgmentOnValueTree(jovts)
for jmtString, targetIx in jovMappings.iteritems():
testJovt = ForaNative.parseStringToJOVT(jmtString)
searchedIx = tree.searchForJOVT(testJovt)
self.assertEqual(targetIx, searchedIx,
"JOV Tree:\n%s\nError: %s mapped to %s instead of %s" % (
str(tree),
str(testJovt),
searchedIx,
targetIx
))
def testMatchStructureCantMatchTooSmallTuple(self):
runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()
jov = FORANative.parseStringToJOV(
"((1,2,3),`StructureMatch,((nothing, false), (nothing, false), (nothing, false), (nothing, false)))"
)
axiom = axioms.getAxiomByJOVT(runtime.getTypedForaCompiler(),
jov.asTuple.jov)
self.assertTrue(
list(axiom.asNative.resultSignature.resultPart().vals) ==
[FORANative.parseStringToJOV("nothing")])
def test_teardown_simple_2(self):
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInternalSplitting=False)
context.placeInEvaluationState(
FORANative.ImplValContainer((FORA.extractImplValContainer(
FORA.eval(
"fun(){ let f = fun() { throw 1 }; try { f() } catch(...) { throw 2 } }"
)), FORANative.symbol_Call)))
context.compute()
self.assertTrue(context.getFinishedResult().isException())
def test_verifySmallAllocNotRemapping(self):
vdmm = ForaNative.createTestingVDMM()
ForaNative.executeBigVectorHandleAllocationsForTimePeriod(
vdmm, .25, 1024 * 1024)
bytes1 = vdmm.totalBytesMmappedCumulatively()
ForaNative.executeBigVectorHandleAllocationsForTimePeriod(
vdmm, 1.0, 1024 * 1024)
bytes2 = vdmm.totalBytesMmappedCumulatively()
self.assertTrue(bytes2 <= 1024 * 1024 * 4)
def loadEntireFileDataset(datasetDescriptor, vdid, vdm):
try:
vectorSlices = produceVDIDSlicesForFile(
datasetDescriptor.asEntireFileDataset.file)
vector = ForaNative.createFORAFreeBinaryVectorFromSlices(
vectorSlices, vdm)
if not vdm.loadImplvalIntoUnloadedVectorHandle(vdid, vector):
raise DatasetLoadException("Couldn't load dataset into VDM")
except InvalidDatasetException as e:
logging.error("Failed to load dataset: %s", e.message)
if not vdm.loadImplvalIntoUnloadedVectorHandle(
vdid, ForaNative.ImplValContainer(e.message)):
raise DatasetLoadException("Couldn't load dataset into VDM")
def test_resolveAxiomDirectly_VeryLongComputation(self):
vectorIVC = FORA.extractImplValContainer(FORA.eval("[]"))
jov = ForaNative.parseStringToJOV(("({Vector([])}, `append, 2)"))
joa = self.axioms.resolveAxiomDirectly(self.compiler, jov.getTuple())
self.assertEqual(len(joa.throwPart()), 0)
self.assertEqual(len(joa.resultPart()), 1)
result = joa.resultPart()[0]
self.assertEqual(result,
ForaNative.parseStringToJOV("{Vector([{Int64}])}"))
def extractPagedUnloadedVector(self, vdm, count):
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInternalSplitting=False)
evaluate(
context,
FORA.extractImplValContainer(
FORA.eval("fun() { Vector.range(%s).paged }" % count)),
FORANative.ImplValContainer(FORANative.makeSymbol("Call")))
pagedVec = context.getFinishedResult().asResult.result
vdm.unloadAllPossible()
return pagedVec
def __init__(self, x=None):
object.__init__(self)
if x is None:
self.implVal_ = ForaNative.nothing
elif isinstance(x, FORAValue):
self.implVal_ = x.implVal_
elif isinstance(x, ForaNative.ImplValContainer):
self.implVal_ = x
elif isinstance(x, tuple):
self.implVal_ = ForaNative.ImplValContainer(
tuple([FORAValue(elt).implVal_ for elt in x]))
else:
#this will attempt to convert numerics
#and will otherwise represent the value as a python object
self.implVal_ = ForaNative.ImplValContainer(x)
def loadEntireS3Dataset(datasetDescriptor, s3InterfaceFactory, vdid, vdm):
try:
vectorSlices = produceVDIDSlicesForEntireS3Dataset(
s3InterfaceFactory, datasetDescriptor.asEntireS3Dataset.dataset,
vdm)
vector = ForaNative.createFORAFreeBinaryVectorFromSlices(
vectorSlices, vdm)
if not vdm.loadImplvalIntoUnloadedVectorHandle(vdid, vector):
raise DatasetLoadException("Couldn't load dataset into VDM")
except InvalidDatasetException as e:
if not vdm.loadImplvalIntoUnloadedVectorHandle(
vdid, ForaNative.ImplValContainer(e.message)):
raise DatasetLoadException("Couldn't load dataset into VDM")
def submitComputationOnClient(self, clientIndex, expressionText, **freeVariables):
if isinstance(expressionText, ForaNative.ImplValContainer):
computationDefinition = CumulusNative.ComputationDefinition.Root(
CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm(
[CumulusNative.ComputationDefinitionTerm.Value(x, None) for x in expressionText.getTuple()]
)
)
elif isinstance(expressionText, CumulusNative.ComputationDefinition):
computationDefinition = expressionText
else:
varNames = list(freeVariables.keys())
expr = FORA.eval("fun(" + ",".join(varNames) + ") {" + expressionText + " } ")
computationDefinition = createComputationDefinition(
FORA.extractImplValContainer(
expr
),
ForaNative.makeSymbol("Call"),
*[freeVariables[v] for v in varNames]
)
computationId = self.getClient(clientIndex).createComputation(computationDefinition)
self.getClient(clientIndex).setComputationPriority(
computationId,
CumulusNative.ComputationPriority(1)
)
return computationId
def runTest():
results.append(
ForaNative.executeMemoryUpdatesForTimePeriod(
measurementTime,
blockSize
)
)
def makeComputationDefinitionFromIVCs(*args):
return CumulusNative.ComputationDefinition.Root(
CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm(
[CumulusNative.ComputationDefinitionTerm.Value(ForaNative.ImplValContainer(x), None)
for x in args]
)
)
def test_sortManySmallVectors(self):
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let shouldAllBeTrue =
Vector.range(20, fun(o) {
sorting.isSorted(
sort(Vector.range(50000 + o, fun(x) { x / 10 }))
)
});
for s in shouldAllBeTrue {
if (not s)
return false
}
return true
}"""
)
)
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == True, response[1].asResult.result.pyval)
def test_expensive_brownian(self):
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let brownian = fun(x,t) {
if (t == 0)
return sum(0.0, x * 10.0**7.0)
else
{
let (l,r) = cached(brownian(x - 1, t - 1), brownian(x + 1, t - 1));
return sum(0.0, x * 10.0**7.0) + l + r
}
}
brownian(0, 5)
}"""
)
)
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
self.assertTrue(response[1].isResult())
def test_manyDuplicateCachecallsAndAdding(self):
self.desirePublisher.desireNumberOfWorkers(2, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
Vector.range(20).papply(fun(x) {
x + cached(sum(0,10**11))[0]
})
}"""
)
)
computationId = self.gateway.requestComputation(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
time.sleep(5)
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
try:
response = self.gateway.finalResponses.get(timeout=240.0)
except Queue.Empty:
response = None
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.gateway.deprioritizeComputation(computationId)
def test_sortALargeVectorWithFourWorkers(self):
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let v = Vector.range(12500000, {_%3.1415926});
if (sorting.isSorted(sorting.sort(v))) 'sorted'
}"""
)
)
try:
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
),
timeout=120.0
)
except Queue.Empty:
response = None
if response is None:
try:
dumpFun = self.dumpSchedulerEventStreams
dumpFun()
except:
logging.warn("Wanted to dump CumulusWorkerEvents, but couldn't");
self.assertTrue(response is not None)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == 'sorted', response[1].asResult.result.pyval)
def measureMmapPerformance(self, measurementTime, blockSize, threadCount, initializeData):
t0 = time.time()
results = []
vdmm = ForaNative.createTestingVDMM()
def runTest():
results.append(
ForaNative.executeMMapAndFreeForTimePeriod(
vdmm,
measurementTime,
blockSize,
initializeData
)
)
threads = [threading.Thread(target=runTest) for ix in range(threadCount)]
for t in threads:
t.start()
for t in threads:
t.join()
return (time.time() - t0) / (sum(results) / 1024.0 ** 3)
def produceVDIDSlicesForSingleBucketKeyPair(s3Interface, bucketname, keyname, s3Dataset):
try:
totalBytes = s3Interface.getKeySize(bucketname, keyname)
chunks = getAppropriateChunksForSize(totalBytes, CHUNK_SIZE)
slices = []
for lowIndex, highIndex in chunks:
externalDatasetDesc = ForaNative.ExternalDatasetDescriptor.S3DatasetSlice(
s3DatasetWithKeyname(s3Dataset, keyname),
lowIndex,
highIndex
)
vectorDataId = ForaNative.VectorDataID.External(externalDatasetDesc)
vectorDataIdSlice = ForaNative.createVectorDataIDSlice(vectorDataId,
0,
highIndex - lowIndex)
slices.append(vectorDataIdSlice)
return slices
except S3Interface.S3InterfaceError as e:
message = "Error loading S3 dataset: %s/%s:\n%s" % (
bucketname,
keyname,
e
)
logging.error(message)
raise InvalidDatasetException(message)
def reasonAboutExpression(self, expression, **variableJudgments):
reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.axioms, False)
keys = sorted(list(variableJudgments.keys()))
functionText = "fun(" + ",".join(['_'] + keys) + ") { " + expression + " }"
frame = reasoner.reason(
makeJovt(
FORANative.parseStringToJOV(functionText),
*[FORANative.parseStringToJOV(variableJudgments[k]) for k in keys]
)
)
self.dumpReasonerSummary(reasoner, frame)
return frame
def convertClassInstanceDescription(self, objectId, classInstanceDescription, convertedValues):
classMemberNameToImplVal = {
classMemberName: convertedValues[memberId]
for classMemberName, memberId in
classInstanceDescription.classMemberNameToClassMemberId.iteritems()
}
classImplVal = convertedValues[classInstanceDescription.classId]
if classImplVal.isSymbol():
convertedValues[objectId] = classImplVal
return
memberNames = tuple(sorted(name for name in classMemberNameToImplVal.iterkeys()))
memberValues = tuple(classMemberNameToImplVal[name] for name in memberNames)
convertedValueOrNone = ForaNative.simulateApply(
ForaNative.ImplValContainer(
(classImplVal,
Symbol_CreateInstance,
ForaNative.CreateNamedTuple(memberValues, memberNames))
)
)
if convertedValueOrNone is None:
raise pyfora.PythonToForaConversionError(
("An internal error occurred: " +
"function stage 1 simulation unexpectedly returned None")
)
convertedValues[objectId] = convertedValueOrNone
def __init__(self,
nativeConstantConverter,
nativeDictConverter,
nativeTupleConverter,
nativeListConverter,
vdmOverride,
builtinMemberMapping,
purePythonModuleImplVal):
self.boundExpressions = {}
self.constantConverter = ConstantConverter.ConstantConverter(
nativeConstantConverter=nativeConstantConverter
)
self.nativeDictConverter = nativeDictConverter
self.nativeTupleConverter = nativeTupleConverter
self.nativeListConverter = nativeListConverter
self.vdm_ = vdmOverride
self.nativeConverter = ForaNative.makePythonAstConverter(
nativeConstantConverter,
nativeListConverter,
nativeTupleConverter,
nativeDictConverter,
purePythonModuleImplVal,
builtinMemberMapping
)
def test_cfgSplitting_7(self):
funString = "fun(f) { f(1) + f(2) }"
cfg = self.parseStringToFunction(funString).toCFG(1)
splits = ForaNative.splitControlFlowGraph(cfg, "block_0Let")
self.assertTrue(splits is not None)
def assertFrameHasResultJOV(self, frame, resultJOV):
if isinstance(resultJOV, str):
resultJOV = FORANative.parseStringToJOV(resultJOV)
self.assertTrue(
len(frame.exits().resultPart().vals) == 1, frame.exits())
self.assertEqual(frame.exits().resultPart()[0], resultJOV)
def loadAxiomSignaturesFromFile(self, axioms_file):
"""
Loads axiom signatures from a file `axioms_file`. This file should contain
axiom signatures on each line, but also can have commented lines (starting with a `#`)
or commented blocks (lines surrounded by lines of `\"""`), as in python syntax.
"""
axiom_signatures = []
line_number = 0
in_comment = False
assert os.path.exists(axioms_file), (
"unable to open file `%s'\n"
%axioms_file
)
with open(axioms_file) as f:
for line in f:
line_number += 1
if re.search('^"""', line):
in_comment = (in_comment != True) # in_comment = in_comment XOR True
continue
if not in_comment and not re.search('^#', line) and len(line.strip()) > 0:
try:
axiom_signatures.append(FORANative.parseStringToJOV(
line.strip()).getTuple())
except Exception as inst:
logging.warn("unable to parse to JOV: `%s', %s:%s",
line.strip(), axioms_file, line_number)
raise inst
return axiom_signatures
def test_vecWithinVec(self):
self.desirePublisher.desireNumberOfWorkers(4, blocking=True)
expr = FORA.extractImplValContainer(
FORA.eval(
"""fun() {
let v =
Vector.range(10) ~~ fun(x) {
Vector.range(1000000 + x).paged
};
v = v.paged;
let res = ()
for elt in v
res = res + (elt,)
res..apply(fun(tupElt) { tupElt.sum() })
'got to the end'
}"""
)
)
response = self.evaluateWithGateway(
makeComputationDefinitionFromIVCs(
expr,
ForaNative.makeSymbol("Call")
)
)
self.assertTrue(response[1].isResult())
self.assertTrue(response[1].asResult.result.pyval == 'got to the end', response[1].asResult.result.pyval)
def sim(memberName):
return ForaNative.simulateApply(
ForaNative.ImplValContainer(
(ivc, makeSymbolIvc("Member"),
makeSymbolIvc(memberName))
)
).pyval
