def initialize(setupObjectToUse=None, useLocalEvaluator=True):
global _builtin
if _builtin is not None:
return
Runtime.initialize(setupObjectToUse)
ModuleImporter.initialize(setupObjectToUse)
Evaluator.initialize(setupObjectToUse, useLocalEvaluator)
_builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
def main(parsedArguments):
Runtime.initialize()
testMissingMapFile()
testMissingDatFiles()
testMissingIndexFiles()
testCorruptMapFile()
testCorruptDatFiles()
testCorruptIndexFiles()
def main(parsedArguments):
Runtime.initialize()
testMissingMapFile()
testMissingDatFiles()
testMissingIndexFiles()
testCorruptMapFile()
testCorruptDatFiles()
testCorruptIndexFiles()
def initialize(setupObjectToUse = None, useLocalEvaluator = True):
global _builtin
if _builtin is not None:
return
Runtime.initialize(setupObjectToUse)
ModuleImporter.initialize(setupObjectToUse)
Evaluator.initialize(setupObjectToUse, useLocalEvaluator)
_builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
def main(argv):
t0 = time.time()
Runtime.initialize(Setup.defaultSetup())
runtime = Runtime.getMainRuntime()
handler = runtime.getInterpreterTraceHandler()
handler.replayTracesFromFile(argv[1])
print "took ", time.time() - t0
def main(argv): t0 = time.time() Runtime.initialize(Setup.defaultSetup()) runtime = Runtime.getMainRuntime() handler = runtime.getInterpreterTraceHandler() handler.replayTracesFromFile(argv[1]) print "took ", time.time() - t0
def main(parsedArguments):
Runtime.initialize()
tester = AxiomTester(
parsedArguments.numRandVals,
parsedArguments.numRelaxations,
parsedArguments.maxForRelax,
parsedArguments.maxForRand,
parsedArguments.testAxiomsPath,
parsedArguments.seed
)
tester.test_axiom_consistency()
def test_extractPausedComputationDuringVectorLoad(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
)
pagedVec = context.getFinishedResult().asResult.result
context.placeInEvaluationState(
FORANative.ImplValContainer(
(pagedVec,
FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
FORANative.ImplValContainer(0))
)
)
vdm.unloadAllPossible()
context.resume()
self.assertTrue(context.isVectorLoad())
computation = context.extractPausedComputation()
self.assertEqual(len(computation.frames),1)
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.compiler = self.runtime.getTypedForaCompiler()
self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(
FORA.builtin().implVal_)
def __init__(self, numRandVals, numRelaxations, maxForRelax, maxForRand, testAxiomsPath, seed):
object.__init__(self)
self.callbackScheduler = CallbackScheduler.singletonForTesting()
self.callbackSchedulerFactory = self.callbackScheduler.getFactory()
self.numRandVals = numRandVals
self.numRelaxations = numRelaxations
self.maxForRelax = maxForRelax
self.maxForRand = maxForRand
self.seed = seed
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.typed_fora_compiler = self.runtime.getTypedForaCompiler()
if testAxiomsPath is not None:
pathToUse = testAxiomsPath
else:
pathToUse = UNIT_TEST_AXIOMS_PATH
self.axiom_signatures_to_test = self.loadAxiomSignaturesFromFile(pathToUse)
self.axiom_groups = []
for i in range(self.axioms.axiomCount):
self.axiom_groups.append(self.axioms.getAxiomGroupByIndex(i))
self.symbol_strings = self.loadSymbolStrings()
numpy.random.seed(seed)
def test_extractPausedComputationDuringVectorLoad(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call")))
pagedVec = context.getFinishedResult().asResult.result
context.placeInEvaluationState(
FORANative.ImplValContainer(
(pagedVec,
FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
FORANative.ImplValContainer(0))))
vdm.unloadAllPossible()
context.resume()
self.assertTrue(context.isVectorLoad())
computation = context.extractPausedComputation()
self.assertEqual(len(computation.frames), 1)
def test_resumePausedComputationWithResult(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm,
allowInterpreterTracing=False,
allowInternalSplitting=False)
text = """
let f = fun(v, ix) {
if (ix > 0)
{
let (v2,res) = f(v,ix-1);
return (v2, res + v2[0])
}
`TriggerInterruptForTesting()
return (v, 0)
};
f([1], 10)
"""
evaluate(
context,
FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call")))
assert context.isInterrupted()
pausedComp = context.extractPausedComputation()
framesToUse = pausedComp.asThread.computation.frames[0:5]
pausedComp2 = FORANative.PausedComputationTree(
FORANative.PausedComputation(
framesToUse,
FORA.extractImplValContainer(
FORA.eval("([2], 0)", keepAsForaValue=True)), False))
context.resumePausedComputation(pausedComp2)
context.copyValuesOutOfVectorPages()
context.pageLargeVectorHandles(0)
context.resetInterruptState()
context.compute()
self.assertTrue(context.isFinished())
result = context.getFinishedResult()
self.assertTrue(result.asResult.result[1].pyval == 6)
def testMatchStructureCantMatchTooSmallTuple(self):
runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()
jov = FORANative.parseStringToJOV("((1,2,3),`StructureMatch,((nothing, false), (nothing, false), (nothing, false), (nothing, false)))")
axiom = axioms.getAxiomByJOVT(runtime.getTypedForaCompiler(), jov.asTuple.jov)
self.assertTrue(list(axiom.asNative.resultSignature.resultPart().vals) == [FORANative.parseStringToJOV("nothing")])
def test_resumePausedComputationWithResult(self):
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
text = """
let f = fun(v, ix) {
if (ix > 0)
{
let (v2,res) = f(v,ix-1);
return (v2, res + v2[0])
}
`TriggerInterruptForTesting()
return (v, 0)
};
f([1], 10)
"""
context.evaluate(
FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")),
FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
)
assert context.isInterrupted()
pausedComp = context.extractPausedComputation()
framesToUse = pausedComp.frames[0:5]
pausedComp2 = FORANative.PausedComputation(
framesToUse,
FORA.extractImplValContainer(FORA.eval("([2], 0)", keepAsForaValue=True)),
False
)
context.resumePausedComputation(pausedComp2)
context.copyValuesOutOfVectorPages()
context.pageLargeVectorHandles(0)
context.resetInterruptState()
context.resume()
self.assertTrue( context.isFinished() )
result = context.getFinishedResult()
self.assertTrue(result.asResult.result[1].pyval == 6)
def testAxiomLookups(self):
axioms = Runtime.getMainRuntime().getAxioms()
jmtsToTest = [axiom.signature() for axiom in axioms]
for tupJmt in jmtsToTest:
self.assertEqual(
axioms.axiomSearchLinear(tupJmt),
axioms.axiomSearchTree(tupJmt),
"Didn't produce same results for %s" % str(tupJmt))
def testAxiomLookups(self):
axioms = Runtime.getMainRuntime().getAxioms()
jmtsToTest = [axiom.signature() for axiom in axioms]
for tupJmt in jmtsToTest:
self.assertEqual(
axioms.axiomSearchLinear(tupJmt),
axioms.axiomSearchTree(tupJmt),
"Didn't produce same results for %s" % str(tupJmt)
)
def triggerCompilation(self, *args):
runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()
compiler = runtime.getTypedForaCompiler()
instructionGraph = runtime.getInstructionGraph()
reasoner = ForaNative.SimpleForwardReasoner(compiler, instructionGraph, axioms)
frame = reasoner.reasonAboutComputationDefinition(self.cumulusComputationDefinition)
logging.critical("Result was %s", frame.exits())
def runSimpleEvaluation(self, inputType, outputType):
mainRuntime = Runtime.getMainRuntime()
foraCompiler = mainRuntime.getTypedForaCompiler()
while foraCompiler.anyCompilingOrPending():
time.sleep(.01)
aParticularStringValue = ForaNative.ImplValContainer(aBigString)
callable = self.generateSimpleCallable(inputType, outputType)
jumpTarget = foraCompiler.compile(callable)
import gc
gc.collect()
for passIndex in range(PASS_COUNT):
#type values are memoized, so we can't assume that the value has a refcount
# of exactly one
totalStringCount = ForaNative.totalStringCount()
totalImplvalCount = ForaNative.totalImplvalCount()
anExecutionContext = ExecutionContext.ExecutionContext(
dataManager = VectorDataManager.constructVDM(self.callbackScheduler)
)
anExecutionContext.evaluateFunctionPointer(jumpTarget, aParticularStringValue)
self.assertTrue(anExecutionContext.isFinished())
res = anExecutionContext.getFinishedResult()
self.assertTrue(not res.isException())
self.assertEqual(res.asResult.result, aParticularStringValue)
anExecutionContext.teardown()
res = None
#verify final refcounts
self.assertEqual(
aParticularStringValue.getStringObjectRefcount(),
1,
"refcounts weren't maintained in %s->%s. %s != 1" % (
inputType,
outputType,
aParticularStringValue.getStringObjectRefcount()
)
)
self.assertEqual(
(totalStringCount, totalImplvalCount),
(ForaNative.totalStringCount(), ForaNative.totalImplvalCount()),
"refcounts weren't maintained in " + str(inputType) + "->" + str(outputType)
)
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.compiler = self.runtime.getTypedForaCompiler()
self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
pyforaPath = os.path.join(os.path.split(pyfora.__file__)[0], "fora/purePython")
self.purePythonAsJOV = FORANative.JudgmentOnValue.Constant(FORA.importModule(pyforaPath).implVal_)
self.instructionGraph = self.runtime.getInstructionGraph()
self.reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.instructionGraph, self.axioms)
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.compiler = self.runtime.getTypedForaCompiler()
self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
pyforaPath = os.path.join(os.path.split(pyfora.__file__)[0], "fora/purePython")
self.purePythonAsJOV = FORANative.JudgmentOnValue.Constant(FORA.importModule(pyforaPath).implVal_)
self.instructionGraph = self.runtime.getInstructionGraph()
self.reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.instructionGraph, self.axioms)
def reasonAboutForaCode(module, testName):
if reasoner[0] is None:
runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()
compiler = runtime.getTypedForaCompiler()
reasoner[0] = ForaNative.SimpleForwardReasoner(compiler, axioms, False)
moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)
frame = reasoner[0].reason(makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))
dumpReasonerSummary(reasoner[0], frame)
return True
def triggerCompilation(self, *args):
runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()
compiler = runtime.getTypedForaCompiler()
instructionGraph = runtime.getInstructionGraph()
reasoner = ForaNative.SimpleForwardReasoner(compiler, instructionGraph,
axioms)
frame = reasoner.reasonAboutComputationDefinition(
self.cumulusComputationDefinition)
logging.critical("Result was %s", frame.exits())
def runSimpleEvaluation(self, inputType, outputType):
mainRuntime = Runtime.getMainRuntime()
foraCompiler = mainRuntime.getTypedForaCompiler()
while foraCompiler.anyCompilingOrPending():
time.sleep(.01)
aParticularStringValue = ForaNative.ImplValContainer(aBigString)
callable = self.generateSimpleCallable(inputType, outputType)
jumpTarget = foraCompiler.compile(callable)
import gc
gc.collect()
for passIndex in range(PASS_COUNT):
#type values are memoized, so we can't assume that the value has a refcount
# of exactly one
totalStringCount = ForaNative.totalStringCount()
totalImplvalCount = ForaNative.totalImplvalCount()
anExecutionContext = ExecutionContext.ExecutionContext(
dataManager=VectorDataManager.constructVDM(
self.callbackScheduler))
anExecutionContext.evaluateFunctionPointer(jumpTarget,
aParticularStringValue)
self.assertTrue(anExecutionContext.isFinished())
res = anExecutionContext.getFinishedResult()
self.assertTrue(not res.isException())
self.assertEqual(res.asResult.result, aParticularStringValue)
anExecutionContext.teardown()
res = None
#verify final refcounts
self.assertEqual(
aParticularStringValue.getStringObjectRefcount(), 1,
"refcounts weren't maintained in %s->%s. %s != 1" %
(inputType, outputType,
aParticularStringValue.getStringObjectRefcount()))
self.assertEqual((totalStringCount, totalImplvalCount),
(ForaNative.totalStringCount(),
ForaNative.totalImplvalCount()),
"refcounts weren't maintained in " +
str(inputType) + "->" + str(outputType))
def reasonAboutForaCode(module, testName):
if reasoner[0] is None:
runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()
compiler = runtime.getTypedForaCompiler()
reasoner[0] = ForaNative.SimpleForwardReasoner(compiler, axioms, False)
moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)
frame = reasoner[0].reason(
makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))
dumpReasonerSummary(reasoner[0], frame)
return True
def setUp(self):
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.native_runtime = self.runtime.getTypedForaCompiler()
symbol_strings_set = Set()
self.symbol_strings = []
for i in range(self.axioms.axiomCount):
sig = str(self.axioms.getAxiomGroupByIndex(i).signature())
for j in [l.strip(' ') for l in sig.strip("()").split(',')]:
if re.search('^`', j):
if j not in symbol_strings_set:
symbol_strings_set.add(j)
self.symbol_strings.append(j.strip('`'))
def testMatchStructureCantMatchTooSmallTuple(self):
runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()
jov = FORANative.parseStringToJOV(
"((1,2,3),`StructureMatch,((nothing, false), (nothing, false), (nothing, false), (nothing, false)))"
)
axiom = axioms.getAxiomByJOVT(runtime.getTypedForaCompiler(),
jov.asTuple.jov)
self.assertTrue(
list(axiom.asNative.resultSignature.resultPart().vals) ==
[FORANative.parseStringToJOV("nothing")])
def reasonAboutForaCode(module, moduleName, testName):
if reasoner[0] is None:
runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()
compiler[0] = runtime.getTypedForaCompiler()
instructionGraph = runtime.getInstructionGraph()
reasoner[0] = ForaNative.SimpleForwardReasoner(compiler[0], instructionGraph, axioms)
moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)
t0 = time.time()
frame = reasoner[0].reasonAboutApply(makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))
print "reasoning ", moduleName + "." + testName, " took ", time.time() - t0
dumpReasonerSummary(moduleName, testName, reasoner[0], frame)
return True
def test_extractStacktrace(self):
text = """fun() {
let x = 0;
let v = [1,2,3]
while (x < 100000) {
x = x + 1
}
(x,v)
}"""
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
context.interruptAfterCycleCount(1000)
context.evaluate(
FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call
)
trace = context.extractStacktrace(True)
self.assertTrue(len(trace) == 1)
stacktrace = trace[0][0]
frameData = trace[0][1]
codeLocation = FORANative.getCodeLocation(stacktrace.getIDs()[0])
self.assertTrue(codeLocation is not None)
self.assertTrue(frameData.wasCompiled is False)
self.assertTrue(len(frameData.inScopeValues) == 2)
self.assertTrue([str(x) for x in frameData.inScopeValues.keys] == ['x', 'v'])
values = [frameData.inScopeValues[x] for x in frameData.inScopeValues.keys]
self.assertTrue(values[0].isInt64())
self.assertTrue(values[1].isVector())
context.teardown()
def test_extractStacktrace(self):
text = """fun() {
let x = 0;
let v = [1,2,3]
while (x < 100000) {
x = x + 1
}
(x,v)
}"""
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.interruptAfterCycleCount(1000)
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
trace = context.extractStacktrace(True)
self.assertTrue(len(trace) == 1)
stacktrace = trace[0][0]
frameData = trace[0][1]
codeLocation = FORANative.getCodeLocation(stacktrace.getIDs()[0])
self.assertTrue(codeLocation is not None)
self.assertTrue(frameData.wasCompiled is False)
self.assertTrue(len(frameData.inScopeValues) == 2)
self.assertTrue([str(x)
for x in frameData.inScopeValues.keys] == ['x', 'v'])
values = [
frameData.inScopeValues[x] for x in frameData.inScopeValues.keys
]
self.assertTrue(values[0].isInt64())
self.assertTrue(values[1].isVector())
context.teardown()
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.native_runtime = self.runtime.getTypedForaCompiler()
self.vals_to_test = self.loadValuesFromFile(os.path.join(os.path.split(__file__)[0],
"AxiomJOA_test.txt"))
self.evaluator = LocalEvaluator.LocalEvaluator(
lambda vdm: CumulusNative.SimpleOfflineCache(self.callbackScheduler, 1000000000),
10000000,
maxPageSizeInBytes = 100000
)
self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)
self.knownModulesAsConstantJOVs = dict()
self.knownModulesAsConstantJOVs["builtin"] = \
FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.native_runtime = self.runtime.getTypedForaCompiler()
self.vals_to_test = self.loadValuesFromFile(
os.path.join(os.path.split(__file__)[0], "AxiomJOA_test.txt"))
self.evaluator = LocalEvaluator.LocalEvaluator(
lambda vdm: CumulusNative.SimpleOfflineCache(
self.callbackScheduler, 1000000000),
10000000,
maxPageSizeInBytes=100000)
self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)
self.knownModulesAsConstantJOVs = dict()
self.knownModulesAsConstantJOVs["builtin"] = \
FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
def runSimpleEvaluation(self, callable, arguments, validator):
mainRuntime = Runtime.getMainRuntime()
foraCompiler = mainRuntime.getTypedForaCompiler()
jumpTarget = foraCompiler.compile(callable)
while foraCompiler.anyCompilingOrPending():
time.sleep(.01)
gc.collect()
for passIndex in range(PASS_COUNT):
totalStringCount = ForaNative.totalStringCount()
totalImplvalCount = ForaNative.totalImplvalCount()
anExecutionContext = ExecutionContext.ExecutionContext(
dataManager = VectorDataManager.constructVDM(self.callbackScheduler)
)
anExecutionContext.evaluateFunctionPointer(jumpTarget, *arguments)
self.assertTrue(anExecutionContext.isFinished())
res = anExecutionContext.getFinishedResult()
self.assertTrue(validator(res.asResult.result),
"invalid result in " + str(callable) + " with " + str(arguments) +
". got " + str(res)
)
res = None
anExecutionContext.teardown()
curRefs = (ForaNative.totalStringCount(), ForaNative.totalImplvalCount())
self.assertEqual(
(totalStringCount, totalImplvalCount),
curRefs,
"refcounts weren't maintained in " + str(callable) + " with " + str(arguments) +
". %s != %s" % (
(totalStringCount, totalImplvalCount),
curRefs
)
)
def test_extractPausedComputation(self):
text = """fun() {
let x = 0;
while (x < 100000)
x = x + 1
x
}"""
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
context.interruptAfterCycleCount(1010)
context.evaluate(
FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call
)
computation = context.extractPausedComputation()
self.assertEqual(len(computation.frames), 2)
self.assertEqual(computation.frames[1].values[0].pyval, 335)
context2 = ExecutionContext.ExecutionContext(
dataManager = vdm,
allowInterpreterTracing = False
)
context2.resumePausedComputation(computation)
context2.resume()
self.assertEqual(context2.getFinishedResult().asResult.result.pyval, 100000)
context.teardown()
context2.teardown()
def reasonAboutForaCode(module, moduleName, testName):
if reasoner[0] is None:
runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()
compiler[0] = runtime.getTypedForaCompiler()
instructionGraph = runtime.getInstructionGraph()
reasoner[0] = ForaNative.SimpleForwardReasoner(compiler[0],
instructionGraph,
axioms)
moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)
t0 = time.time()
frame = reasoner[0].reasonAboutApply(
makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))
print "reasoning ", moduleName + "." + testName, " took ", time.time() - t0
dumpReasonerSummary(moduleName, testName, reasoner[0], frame)
return True
def test_extractPausedComputation(self):
text = """fun() {
let x = 0;
while (x < 100000)
x = x + 1
x
}"""
self.runtime = Runtime.getMainRuntime()
#self.dynamicOptimizer = self.runtime.dynamicOptimizer
vdm = FORANative.VectorDataManager(callbackScheduler,
Setup.config().maxPageSizeInBytes)
context = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context.interruptAfterCycleCount(1010)
context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
FORANative.symbol_Call)
computation = context.extractPausedComputation()
self.assertEqual(len(computation.frames), 2)
self.assertEqual(computation.frames[1].values[0].pyval, 335)
context2 = ExecutionContext.ExecutionContext(
dataManager=vdm, allowInterpreterTracing=False)
context2.resumePausedComputation(computation)
context2.resume()
self.assertEqual(context2.getFinishedResult().asResult.result.pyval,
100000)
context.teardown()
context2.teardown()
def setUp(self):
self.callbackScheduler = CallbackScheduler.singletonForTesting()
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.compiler = self.runtime.getTypedForaCompiler()
self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
def main(argv): fora.init_local() runtime = Runtime.getMainRuntime() handler = runtime.getInterpreterTraceHandler() handler.replayTracesFromFile(argv[1])
def main(argv):
fora.init_local()
runtime = Runtime.getMainRuntime()
handler = runtime.getInterpreterTraceHandler()
handler.replayTracesFromFile(argv[1])
def __init__(self, callbackScheduler, sharedStateViewFactory, computedValueGatewayFactory):
self.lock = threading.Lock()
self.cacheLoadEvents = {}
self.resultsById_ = {}
self.eventsById_ = {}
logging.info("created a component host")
self.graph = ComputedGraph.ComputedGraph()
logging.info("created a ComputedGraph")
Runtime.initialize()
logging.info("Runtime initialized")
ModuleImporter.initialize()
logging.info("Module importer initialized")
Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
self.incomingObjectCache = IncomingObjectCache()
self.outgoingObjectCache = OutgoingObjectCache()
self.VDM = VectorDataManager.constructVDM(callbackScheduler)
self.VDM.setDropUnreferencedPagesWhenFull(True)
logging.info("created a VDM")
logging.info("got shared state view factory: %s", sharedStateViewFactory)
def initValueGateway():
with self.graph:
self.computedValueGateway = computedValueGatewayFactory()
self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway
def initSynchronizer():
self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
logging.info("created a SharedStateSynchronizer")
self.synchronizer.attachView(sharedStateViewFactory.createView())
logging.info("attached shared state view.")
simultaneously(initSynchronizer, initValueGateway)
self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()
self.outstandingMessagesById = {}
self.expectedMessageId = 0
self.messageTypeHandlers = {}
self.messageTypeHandlers["Read"] = self.handleReadMessage
self.messageTypeHandlers["Assign"] = self.handleAssignMessage
self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds
self.pendingObjectQueue = []
self.subscriptions = Subscriptions.Subscriptions(self.graph, self.computedValueGateway, self.synchronizer)
def __init__(self,
callbackScheduler,
sharedStateViewFactory,
computedValueGatewayFactory):
self.lock = threading.Lock()
self.cacheLoadEvents = {}
self.resultsById_ = {}
self.eventsById_ = {}
logging.info("created a component host")
self.graph = ComputedGraph.ComputedGraph()
logging.info("created a ComputedGraph")
Runtime.initialize()
logging.info("Runtime initialized")
ModuleImporter.initialize()
logging.info("Module importer initialized")
Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
self.incomingObjectCache = IncomingObjectCache()
self.outgoingObjectCache = OutgoingObjectCache()
self.VDM = VectorDataManager.constructVDM(callbackScheduler)
self.VDM.setDropUnreferencedPagesWhenFull(True)
logging.info("created a VDM")
logging.info("got shared state view factory: %s", sharedStateViewFactory)
def initValueGateway():
with self.graph:
self.computedValueGateway = computedValueGatewayFactory()
self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway
def initSynchronizer():
self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
logging.info("created a SharedStateSynchronizer")
self.synchronizer.attachView(
sharedStateViewFactory.createView()
)
logging.info("attached shared state view.")
simultaneously(
initSynchronizer,
initValueGateway
)
self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()
self.outstandingMessagesById = {}
self.expectedMessageId = 0
self.messageTypeHandlers = {}
self.messageTypeHandlers["Read"] = self.handleReadMessage
self.messageTypeHandlers["Assign"] = self.handleAssignMessage
self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds
self.pendingObjectQueue = []
self.subscriptions = Subscriptions.Subscriptions(
self.graph,
self.computedValueGateway,
self.synchronizer
)
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
Run this script to generate a file of axiom signatures which can be fed into Axioms_consistency_test.py.
This generates all axioms, but by default comments them all out. Uncomment the ones you want to test.
"""
import ufora.native
import ufora.FORA.python.Runtime as Runtime
import ufora.native.FORA as FORANative
import os
dir = os.path.dirname(__file__)
AXIOMS_TO_TEST_FILENAME = os.path.join(dir, "AXIOMS_TO_TEST.txt")
runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()
readme_string = '"""\nThis file lists all the axioms we would like to check for consistency in\nAxioms_consistency_test.py. It supports basic python-like commenting\n"""\n\n'
with open(AXIOMS_TO_TEST_FILENAME, "w") as f:
f.write(readme_string)
f.write('"""\n')
for i in range(axioms.axiomCount):
f.write("%s\n" %axioms.getAxiomGroupByIndex(i).signature())
f.write('"""')
def __init__(self, ownAddress, channelListener, channelFactory,
eventHandler, callbackScheduler, diagnosticsDir, config,
viewFactory):
Stoppable.Stoppable.__init__(self)
#acquire a machineId randomly, using uuid
self.machineId = CumulusNative.MachineId(
Hash.Hash.sha1(str(uuid.uuid4())))
self.ownAddress = ownAddress
self.callbackScheduler = callbackScheduler
self.viewFactory = viewFactory
self.threadsStarted_ = False
self.connectedMachines = set()
self.connectingMachines = set(
) # machines we are in the process of connecting to
self.droppedMachineIds = set()
self.lock = threading.RLock()
self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024 * 1024
self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024 * 1024
self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc
self.reconnectPersistentCacheIndexViewThreads = []
if config.cumulusDiskCacheStorageSubdirectory is not None:
self.cumulusDiskCacheWantsDeletionOnTeardown = True
self.cumulusDiskCacheStorageDir = os.path.join(
config.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageSubdirectory)
else:
self.cumulusDiskCacheWantsDeletionOnTeardown = False
self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir
logging.info(
"Creating a CumulusService with ram cache of %s / %s MB and %s threads",
self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusThreadCountOverride)
self._stopEvent = threading.Event()
self._channelListener = channelListener
assert len(self._channelListener.ports) == 2
self._channelFactory = channelFactory
Runtime.initialize()
ModuleImporter.initialize()
self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
self.viewFactory)
self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
target=self._channelListener.start)
self.vdm = VectorDataManager.constructVDM(
callbackScheduler, self.cumulusVectorRamCacheSizeOverride,
self.cumulusMaxRamCacheSizeOverride)
if self.cumulusTrackTcMalloc:
logging.info(
"CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0)
self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()
self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
viewFactory.createView(retrySeconds=10.0, numRetries=10),
callbackScheduler)
self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)
self.deleteCumulusDiskCacheIfNecessary()
self.offlineCache = CumulusNative.DiskOfflineCache(
callbackScheduler, self.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageMB * 1024 * 1024,
config.cumulusDiskCacheStorageFileCount)
checkpointInterval = config.cumulusCheckpointIntervalSeconds
if checkpointInterval == 0:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None ()
else:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
checkpointInterval, 1024 * 1024)
self.cumulusWorker = self.constructCumlusWorker(
callbackScheduler,
CumulusNative.CumulusWorkerConfiguration(
self.machineId,
self.cumulusThreadCountOverride, checkpointPolicy,
ExecutionContext.createContextConfiguration(), diagnosticsDir
or ""), self.vdm, self.offlineCache, eventHandler)
#externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
#callbackScheduler
#)
#self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
#settings.s3InterfaceFactory,
#settings.objectStore,
#self.vdm,
#externalDatasetChannel.makeQueuelike(callbackScheduler)
#)
self.cumulusWorker.startComputations()
def setUp(self):
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.native_runtime = self.runtime.getTypedForaCompiler()
self.vdm = FORANative.VectorDataManager(
CallbackScheduler.singletonForTesting(), 10000000)
def setUp(self):
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.compiler = self.runtime.getTypedForaCompiler()
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
rebuildAxiomSearchFunction
Generate appropriate axiom build functions after we've built the project successfully.
"""
import os
import ufora
import ufora.config.Setup as Setup
import ufora.FORA.python.Runtime as Runtime
Runtime.initialize(Setup.defaultSetup())
axioms = Runtime.getMainRuntime().getAxioms()
implvalCode, jovtCode = axioms.getCppWrapperCode()
uforaDir = os.path.split(os.path.abspath(ufora.__file__))[0]
with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch.cpp"), "w") as f:
print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
print >> f, implvalCode
with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch2.cpp"), "w") as f:
print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
print >> f, jovtCode
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
"""
rebuildAxiomSearchFunction
Generate appropriate axiom build functions after we've built the project successfully.
"""
import os
import ufora
import ufora.config.Setup as Setup
import ufora.FORA.python.Runtime as Runtime
Runtime.initialize(Setup.defaultSetup())
axioms = Runtime.getMainRuntime().getAxioms()
implvalCode, jovtCode = axioms.getCppWrapperCode()
uforaDir = os.path.split(os.path.abspath(ufora.__file__))[0]
with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch.cpp"),
"w") as f:
print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
print >> f, implvalCode
with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch2.cpp"),
"w") as f:
print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
def setUp(self):
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.native_runtime = self.runtime.getTypedForaCompiler()
self.vdm = FORANative.VectorDataManager(CallbackScheduler.singletonForTesting(), 10000000)
def setUp(self):
self.runtime = Runtime.getMainRuntime()
self.axioms = self.runtime.getAxioms()
self.compiler = self.runtime.getTypedForaCompiler()
def __init__(self,
ownAddress,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnosticsDir,
config,
viewFactory,
s3InterfaceFactory=None,
objectStore=None):
Stoppable.Stoppable.__init__(self)
#acquire a machineId randomly, using uuid
self.machineId = CumulusNative.MachineId(
Hash.Hash.sha1(str(uuid.uuid4()))
)
self.ownAddress = ownAddress
self.callbackScheduler = callbackScheduler
self.viewFactory = viewFactory
self.s3InterfaceFactory = s3InterfaceFactory
self.objectStore = objectStore
self.threadsStarted_ = False
self.connectedMachines = set()
self.connectingMachines = set() # machines we are in the process of connecting to
self.droppedMachineIds = set()
self.lock = threading.RLock()
self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
self.cumulusTrackTcmalloc = config.cumulusTrackTcmalloc
self.eventHandler = eventHandler
self.reconnectPersistentCacheIndexViewThreads = []
if config.cumulusDiskCacheStorageSubdirectory is not None:
self.cumulusDiskCacheWantsDeletionOnTeardown = True
self.cumulusDiskCacheStorageDir = os.path.join(
config.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageSubdirectory
)
else:
self.cumulusDiskCacheWantsDeletionOnTeardown = False
self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir
self._stopEvent = threading.Event()
self._channelListener = channelListener
assert len(self._channelListener.ports) == 2
self._channelFactory = channelFactory
Runtime.initialize()
ModuleImporter.initialize()
self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
self.viewFactory
)
self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
target=self._channelListener.start
)
self.vdm = VectorDataManager.constructVDM(
callbackScheduler,
self.cumulusVectorRamCacheSizeOverride,
self.cumulusMaxRamCacheSizeOverride
)
if self.cumulusTrackTcmalloc:
self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()
self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
viewFactory.createView(retrySeconds=10.0, numRetries=10),
callbackScheduler
)
self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)
self.deleteCumulusDiskCacheIfNecessary()
self.offlineCache = CumulusNative.DiskOfflineCache(
callbackScheduler,
self.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageMB * 1024 * 1024,
config.cumulusDiskCacheStorageFileCount
)
#If the "s3InterfaceFactory" is not in-memory, we use real out of process python.
#it would be better if this were more explicit
outOfProcess = self.s3InterfaceFactory is not None and self.s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool
self.outOfProcessPythonTasks = OutOfProcessPythonTasks.OutOfProcessPythonTasks(outOfProcess=outOfProcess)
self.vdm.initializeOutOfProcessPythonTasks(self.outOfProcessPythonTasks.nativeTasks)
checkpointInterval = config.cumulusCheckpointIntervalSeconds
if checkpointInterval == 0:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
else:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
checkpointInterval,
1024 * 1024
)
self.cumulusWorker = self.constructCumlusWorker(
callbackScheduler,
CumulusNative.CumulusWorkerConfiguration(
self.machineId,
self.cumulusThreadCountOverride,
checkpointPolicy,
ExecutionContext.createContextConfiguration(),
diagnosticsDir or ""
),
self.vdm,
self.offlineCache,
eventHandler
)
self.datasetLoadService = None
if self.s3InterfaceFactory:
externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
callbackScheduler
)
self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
self.s3InterfaceFactory,
self.objectStore,
self.vdm,
externalDatasetChannel.makeQueuelike(callbackScheduler)
)
self.cumulusWorker.startComputations()
if self.datasetLoadService:
self.datasetLoadService.startService()
def __init__(self,
ownAddress,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnosticsDir,
config,
viewFactory):
Stoppable.Stoppable.__init__(self)
#acquire a machineId randomly, using uuid
self.machineId = CumulusNative.MachineId(
Hash.Hash.sha1(str(uuid.uuid4()))
)
self.ownAddress = ownAddress
self.callbackScheduler = callbackScheduler
self.viewFactory = viewFactory
self.threadsStarted_ = False
self.connectedMachines = set()
self.connectingMachines = set() # machines we are in the process of connecting to
self.droppedMachineIds = set()
self.lock = threading.RLock()
self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc
self.reconnectPersistentCacheIndexViewThreads = []
if config.cumulusDiskCacheStorageSubdirectory is not None:
self.cumulusDiskCacheWantsDeletionOnTeardown = True
self.cumulusDiskCacheStorageDir = os.path.join(
config.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageSubdirectory
)
else:
self.cumulusDiskCacheWantsDeletionOnTeardown = False
self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir
logging.info(
"Creating a CumulusService with ram cache of %s / %s MB and %s threads",
self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusThreadCountOverride
)
self._stopEvent = threading.Event()
self._channelListener = channelListener
assert len(self._channelListener.ports) == 2
self._channelFactory = channelFactory
Runtime.initialize()
ModuleImporter.initialize()
self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
self.viewFactory
)
self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
target=self._channelListener.start
)
self.vdm = VectorDataManager.constructVDM(
callbackScheduler,
self.cumulusVectorRamCacheSizeOverride,
self.cumulusMaxRamCacheSizeOverride
)
if self.cumulusTrackTcMalloc:
logging.info(
"CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0
)
self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()
self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
viewFactory.createView(retrySeconds=10.0, numRetries=10),
callbackScheduler
)
self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)
self.deleteCumulusDiskCacheIfNecessary()
self.offlineCache = CumulusNative.DiskOfflineCache(
callbackScheduler,
self.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageMB * 1024 * 1024,
config.cumulusDiskCacheStorageFileCount
)
checkpointInterval = config.cumulusCheckpointIntervalSeconds
if checkpointInterval == 0:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
else:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
checkpointInterval,
1024 * 1024
)
self.cumulusWorker = self.constructCumlusWorker(
callbackScheduler,
CumulusNative.CumulusWorkerConfiguration(
self.machineId,
self.cumulusThreadCountOverride,
checkpointPolicy,
ExecutionContext.createContextConfiguration(),
diagnosticsDir or ""
),
self.vdm,
self.offlineCache,
eventHandler
)
#externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
#callbackScheduler
#)
#self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
#settings.s3InterfaceFactory,
#settings.objectStore,
#self.vdm,
#externalDatasetChannel.makeQueuelike(callbackScheduler)
#)
self.cumulusWorker.startComputations()
