def test_cfgSplitting_5(self): funString = "fun(f) { (f(1) + f(2)) + f(3) }" cfg = self.parseStringToFunction(funString).toCFG(1) steps = ForaNative.extractApplyStepsFromControlFlowGraph(cfg, None) splits = ForaNative.splitControlFlowGraph(cfg, "block_0Let") self.assertTrue(splits is not None)
def test_futures_tupleCallInContinuation_2(self): text = "fun(x) { x = x + x; size((x, *x)) }" cfg = self.parseStringToFunction(text).toCFG(1) x = 1 xImplVal = ForaNative.ImplValContainer(x) cfgWithFutures = ForaNative.CFGWithFutures.createCfgWithFutures( cfg, "block_1Let", (xImplVal,)) cfgWithFutures.continueSimulation() cfgWithFutures.continueSimulation() self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [0]) x = x + x cfgWithFutures.slotCompleted( 0, normalComputationResult( ForaNative.ImplValContainer(x) ) ) self.assertFalse(cfgWithFutures.hasResolvedToSimpleState()) self.assertEqual(cfgWithFutures.indicesOfSubmittableFutures(), [1]) self.assertComponentwiseEqual( cfgWithFutures.submittableArgs(1).args.values, [ForaNative.makeSymbol("size"), ForaNative.makeSymbol("Call"), ForaNative.ImplValContainer((x, x))] )
def test_extractPausedComputationDuringVectorLoad(self): self.runtime = Runtime.getMainRuntime() #self.dynamicOptimizer = self.runtime.dynamicOptimizer vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager = vdm, allowInterpreterTracing = False ) context.evaluate( FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")), FORANative.ImplValContainer(FORANative.makeSymbol("Call")) ) pagedVec = context.getFinishedResult().asResult.result context.placeInEvaluationState( FORANative.ImplValContainer( (pagedVec, FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")), FORANative.ImplValContainer(0)) ) ) vdm.unloadAllPossible() context.resume() self.assertTrue(context.isVectorLoad()) computation = context.extractPausedComputation() self.assertEqual(len(computation.frames),1)
def test_joas(self): #"test assertions about JOAs of particular axioms" for item in self.vals_to_test: jovt = self.getJOVT(item[0]) if len(item) == 4: assert item[3] == "hasSideEffects", "illegal third argument to JOA" expected_joa = FORANative.JudgmentOnAction( FORANative.parseStringToJOR(item[1]), FORANative.parseStringToJOR(item[2]), True if len(item) == 4 and item[3] == "hasSideEffects" else False ) computed_joa = self.joa(jovt) if computed_joa: self.assertTrue( expected_joa.resultPart() == computed_joa.resultPart(), "for JOVT %s should have had JOA resultPart: %s, but had: %s" \ % (jovt, expected_joa.resultPart(), computed_joa.resultPart()) ) #be a little more relaxed on the throwParts self.assertTrue( expected_joa.throwPart().covers(computed_joa.throwPart()), "for JOVT %s: expected JOA throwPart %s does not cover computed JOA %s" \ % (jovt, expected_joa.throwPart(), computed_joa.throwPart()) ) if len(expected_joa.throwPart()) > 0: self.assertTrue( len(computed_joa.throwPart()) > 0, "for JOVT %s: expected JOA %s throws, but computed JOA %s does not" \ %(jovt, expected_joa, computed_joa) )
def convertClassInstanceDescription(self, objectId, classInstanceDescription): classMemberNameToImplVal = { classMemberName: self.convertedValues[memberId] for classMemberName, memberId in classInstanceDescription. classMemberNameToClassMemberId.iteritems() } classImplVal = self.convertedValues[classInstanceDescription.classId] if classImplVal.isSymbol(): self.convertedValues[objectId] = classImplVal return memberNames = tuple( sorted(name for name in classMemberNameToImplVal.iterkeys())) memberValues = tuple(classMemberNameToImplVal[name] for name in memberNames) convertedValueOrNone = ForaNative.simulateApply( ForaNative.ImplValContainer( (classImplVal, Symbol_CreateInstance, ForaNative.CreateNamedTuple(memberValues, memberNames)))) if convertedValueOrNone is None: raise pyfora.PythonToForaConversionError( ("An internal error occurred: " + "function stage 1 simulation unexpectedly returned None")) self.convertedValues[objectId] = convertedValueOrNone
def test_createInstance_2(self): classIvc = FORA.eval( """let C = class { member y; member x; }; C""" ).implVal_ x = 1 y = 2 res = ForaNative.simulateApply( ForaNative.ImplValContainer( ( classIvc, makeSymbolIvc("CreateInstance"), ForaNative.ImplValContainer(y), ForaNative.ImplValContainer(x), ) ) ) self.assertIsNotNone(res) computed_x = ForaNative.simulateApply( ForaNative.ImplValContainer((res, self.Symbol_member, makeSymbolIvc("x"))) ) self.assertEqual(computed_x, ForaNative.ImplValContainer(x))
def test_TypedFora_CreateTuple(self): layoutStyles = [ TypedFora.RefcountStyle.Pooled(), TypedFora.RefcountStyle.AsValueUnowned(), TypedFora.RefcountStyle.AsValueOwned() ] jovs = [ ForaNative.parseStringToJOV("{String}"), ForaNative.parseStringToJOV("'a string'"), ForaNative.parseStringToJOV("*") ] aVal = ForaNative.ImplValContainer("a string") allLayouts = [] for ls1 in layoutStyles: for jov1 in jovs: allLayouts.append(TypedFora.Type(jov1, ls1)) for t1 in allLayouts: for t2 in allLayouts: callable = self.generateTupleCallable([t1, t2], [False, False]) def validator(result): return result == ForaNative.ImplValContainer((aVal, aVal)) self.runSimpleEvaluation(callable, [aVal, aVal], validator)
def test_TypedFora_Layout_UsingCompiler(self): layoutStyles = [ TypedFora.RefcountStyle.Pooled(), TypedFora.RefcountStyle.AsValueUnowned(), TypedFora.RefcountStyle.AsValueOwned() ] jovs = [ ForaNative.parseStringToJOV("{String}"), #a small union ForaNative.parseStringToJOV("{Union([{String}, {Int64}])}"), #a much bigger union ForaNative.parseStringToJOV("{Union([{String}, {Int64}, {Float64}, nothing, ({Float64})])}"), ForaNative.parseStringToJOV("'%s'" % aBigString), ForaNative.parseStringToJOV("*") ] for ls1 in layoutStyles: for jov1 in jovs: for ls2 in layoutStyles: for jov2 in jovs: if not (ls1.isAsValueOwned() and ls2.isAsValueUnowned()): t1 = TypedFora.Type(jov1, ls1) t2 = TypedFora.Type(jov2, ls2) self.runSimpleEvaluation(t1, t2)
def copyDataOutOfPagesTest(self, text, cycleCount, expectsToHaveCopies): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False, allowInternalSplitting=False) context.configuration.agressivelyValidateRefcountsAndPageReachability = True context.configuration.releaseVectorHandlesImmediatelyAfterExecution = False context.placeInEvaluationState( FORANative.ImplValContainer( (FORA.extractImplValContainer(FORA.eval(text)), FORANative.symbol_Call))) context.interruptAfterCycleCount(cycleCount) context.compute() if expectsToHaveCopies: self.assertTrue(context.copyValuesOutOfVectorPages()) self.assertFalse(context.copyValuesOutOfVectorPages()) else: self.assertFalse(context.copyValuesOutOfVectorPages())
def createWorkersAndClients(workerCount, clientCount, viewFactory=None, memoryLimitMb=100, threadCount=2, callbackSchedulerToUse=None): if callbackSchedulerToUse is None: callbackSchedulerToUse = CallbackScheduler.singletonForTesting() if viewFactory is None: viewFactory = createInMemorySharedStateViewFactory( callbackSchedulerToUse) workersVdmsAndEventHandlers = [ createWorker(machineId(ix), viewFactory, memoryLimitMb=memoryLimitMb, threadCount=threadCount, callbackSchedulerToUse=callbackSchedulerToUse) for ix in range(workerCount) ] clientsAndVdms = [ createClient(clientId(ix), callbackSchedulerToUse=callbackSchedulerToUse) for ix in range(clientCount) ] for ix1 in range(len(workersVdmsAndEventHandlers)): workersVdmsAndEventHandlers[ix1][0].startComputations() for ix1 in range(len(workersVdmsAndEventHandlers) - 1): for ix2 in range(ix1 + 1, len(workersVdmsAndEventHandlers)): worker1Channel1, worker2Channel1 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) worker1Channel2, worker2Channel2 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) workersVdmsAndEventHandlers[ix1][0].addMachine( machineId(ix2), [worker1Channel1, worker1Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) workersVdmsAndEventHandlers[ix2][0].addMachine( machineId(ix1), [worker2Channel1, worker2Channel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) for ix1 in range(len(workersVdmsAndEventHandlers)): for ix2 in range(len(clientsAndVdms)): workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel( callbackSchedulerToUse) workersVdmsAndEventHandlers[ix1][0].addCumulusClient( clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) clientsAndVdms[ix2][0].addMachine(machineId(ix1), [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), callbackSchedulerToUse) return workersVdmsAndEventHandlers, clientsAndVdms, viewFactory
def test_resumePausedComputationWithResult(self): self.runtime = Runtime.getMainRuntime() #self.dynamicOptimizer = self.runtime.dynamicOptimizer vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInterpreterTracing=False, allowInternalSplitting=False) text = """ let f = fun(v, ix) { if (ix > 0) { let (v2,res) = f(v,ix-1); return (v2, res + v2[0]) } `TriggerInterruptForTesting() return (v, 0) }; f([1], 10) """ evaluate( context, FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")), FORANative.ImplValContainer(FORANative.makeSymbol("Call"))) assert context.isInterrupted() pausedComp = context.extractPausedComputation() framesToUse = pausedComp.asThread.computation.frames[0:5] pausedComp2 = FORANative.PausedComputationTree( FORANative.PausedComputation( framesToUse, FORA.extractImplValContainer( FORA.eval("([2], 0)", keepAsForaValue=True)), False)) context.resumePausedComputation(pausedComp2) context.copyValuesOutOfVectorPages() context.pageLargeVectorHandles(0) context.resetInterruptState() context.compute() self.assertTrue(context.isFinished()) result = context.getFinishedResult() self.assertTrue(result.asResult.result[1].pyval == 6)
def getCacheRequestComputationResult(cacheCallElement): bucketname = cacheCallElement[1].pyvalOrNone keyname = cacheCallElement[2].pyvalOrNone if not isinstance(bucketname, str) or not isinstance(keyname, str): return ForaNative.ComputationResult.Exception( ForaNative.ImplValContainer("Badly formed S3 dataset request: %s" % cacheCallElement) ) s3Interface = getCurrentS3Interface() if s3Interface.keyExists(bucketname, keyname): keysAndSizesMatching = [(keyname, s3Interface.getKeySize(bucketname, keyname))] else: keysAndSizesMatching = s3Interface.listKeysWithPrefix(bucketname, keyname + "_") indicesKeysAndSizes = [] for key, size, mtime in keysAndSizesMatching: try: index = int(key[len(keyname)+1:]) indicesKeysAndSizes.append((index, key, size)) except ValueError: pass keysAndSizesMatching = [(key, size) for _, key, size in sorted(indicesKeysAndSizes)] if not keysAndSizesMatching: return ForaNative.ComputationResult.Exception( ForaNative.ImplValContainer("No keys matching %s/%s using %s" % ( bucketname, keyname, s3Interface )) ) wholeVectorIVC = ForaNative.getEmptyVector() for key, size in keysAndSizesMatching: CHUNK_SIZE = 10 * 1024 * 1024 chunks = getAppropriateChunksForSize(size, CHUNK_SIZE) for lowIndex, highIndex in chunks: externalDatasetDesc = ForaNative.ExternalDatasetDescriptor.S3Dataset( bucketname, key, lowIndex, highIndex ) vectorDataId = ForaNative.VectorDataID.External(externalDatasetDesc) vectorIVC = ForaNative.createFORAFreeBinaryVector(vectorDataId, highIndex - lowIndex) wholeVectorIVC = ForaNative.concatenateVectors(wholeVectorIVC, vectorIVC) return ForaNative.ComputationResult.Result(wholeVectorIVC, ForaNative.ImplValContainer())
def test_serialResults_5(self): text = "fun(x) { x = x + x; size((x, *x)) }" cfg = self.parseStringToFunction(text).toCFG(1) xImplVal = ForaNative.ImplValContainer(1) serialResult = self.serialResult(cfg, (xImplVal, ), "block_1Let") self.assertEqual(serialResult.asResult.result.asResult.result, ForaNative.ImplValContainer(2))
def test_cfgSplitting_1(self): cfg1 = self.parseStringToFunction("fun(f) { f(1) + f(1+3) }").toCFG(1) steps = ForaNative.extractApplyStepsFromControlFlowGraph(cfg1, "block_0Let") self.assertEqual(len(steps), 4) splits = ForaNative.splitControlFlowGraph(cfg1, "block_0Let") self.assertTrue(splits is not None)
def getCacheRequestComputationResult(cacheCallElement): bucketname = cacheCallElement[1].pyvalOrNone keyname = cacheCallElement[2].pyvalOrNone if not isinstance(bucketname, str) or not isinstance(keyname, str): return ForaNative.ComputationResult.Exception( ForaNative.ImplValContainer("Badly formed S3 dataset request: %s" % cacheCallElement)) s3Interface = getCurrentS3Interface() if s3Interface.keyExists(bucketname, keyname): keysAndSizesMatching = [(keyname, s3Interface.getKeySize(bucketname, keyname))] else: keysAndSizesMatching = s3Interface.listKeysWithPrefix( bucketname, keyname + "_") indicesKeysAndSizes = [] for key, size, mtime in keysAndSizesMatching: try: index = int(key[len(keyname) + 1:]) indicesKeysAndSizes.append((index, key, size)) except ValueError: pass keysAndSizesMatching = [ (key, size) for _, key, size in sorted(indicesKeysAndSizes) ] if not keysAndSizesMatching: return ForaNative.ComputationResult.Exception( ForaNative.ImplValContainer("No keys matching %s/%s using %s" % (bucketname, keyname, s3Interface))) wholeVectorIVC = ForaNative.getEmptyVector() for key, size in keysAndSizesMatching: CHUNK_SIZE = 10 * 1024 * 1024 chunks = getAppropriateChunksForSize(size, CHUNK_SIZE) for lowIndex, highIndex in chunks: externalDatasetDesc = ForaNative.ExternalDatasetDescriptor.S3Dataset( bucketname, key, lowIndex, highIndex) vectorDataId = ForaNative.VectorDataID.External( externalDatasetDesc) vectorIVC = ForaNative.createFORAFreeBinaryVector( vectorDataId, highIndex - lowIndex) wholeVectorIVC = ForaNative.concatenateVectors( wholeVectorIVC, vectorIVC) return ForaNative.ComputationResult.Result(wholeVectorIVC)
def testMatchStructureCantMatchTooSmallTuple(self): runtime = Runtime.getMainRuntime() axioms = runtime.getAxioms() jov = FORANative.parseStringToJOV("((1,2,3),`StructureMatch,((nothing, false), (nothing, false), (nothing, false), (nothing, false)))") axiom = axioms.getAxiomByJOVT(runtime.getTypedForaCompiler(), jov.asTuple.jov) self.assertTrue(list(axiom.asNative.resultSignature.resultPart().vals) == [FORANative.parseStringToJOV("nothing")])
def evalSubmittableArgs(submittableArgs): def evalApplyTuple(applyTuple, signature=None): varNames = map(lambda ix: "x_%s" % ix, range(len(applyTuple))) if signature is None: evalStringTerms = varNames else: evalStringTerms = [] for ix in range(len(signature.terms)): term = signature.terms[ix] if term.isNormal(): name = term.asNormal.name if name is not None: evalStringTerms.append("%s: x_%s" % (name, ix)) else: evalStringTerms.append("x_%s" % ix) elif term.isTupleCall(): evalStringTerms.append("*x_%s" % ix) evalString = evalStringTerms[0] + "`(" + ",".join( evalStringTerms[1:]) + ")" try: return normalComputationResult( FORA.extractImplValContainer( FORA.eval(evalString, locals={ name: val for (name, val) in zip(varNames, applyTuple) }, keepAsForaValue=True))) except ForaValue.FORAException as e: return exceptionComputationResult( FORA.extractImplValContainer(e.foraVal)) if submittableArgs.isApply(): return evalApplyTuple(submittableArgs.args.values, submittableArgs.args.signature) else: # submittableArgs.isCached() tr = () # this corresponds to the logic for Cached nodes in # CFGWithFutures::asSubmittable assert len(submittableArgs.args.values) == 1 for cachedPair in submittableArgs.args.values[0]: applyArgs = [cachedPair[0], ForaNative.makeSymbol("Call")] applyArgs.extend(cachedPair[1]) res = evalApplyTuple(applyArgs) if res.isException(): return res tr = tr + (res.asResult.result, ) return normalComputationResult(ForaNative.ImplValContainer(tr))
def test_vectorOfPagedVectorApplyWithDropping(self): self.desirePublisher.desireNumberOfWorkers(3, blocking=True) expr = FORA.extractImplValContainer( FORA.eval( """fun() { let v = Vector.range(20).apply(fun(ix) { Vector.range(1250000.0+ix).paged }).paged let lookup = fun(ix) { v[ix] } Vector.range(100).apply(fun(ix) { sum(0, 10**8); cached(lookup(ix)) }) v }""" ) ) computation1 = self.gateway.requestComputation( makeComputationDefinitionFromIVCs( expr, ForaNative.makeSymbol("Call") ) ) try: response = self.gateway.finalResponses.get(timeout=60.0) except Queue.Empty: response = None self.assertTrue(response is not None, response) self.gateway.deprioritizeComputation(computation1) self.desirePublisher.desireNumberOfWorkers(2, blocking=True) expr2 = FORA.extractImplValContainer( FORA.eval( """fun() { let res = 0 for ix in sequence(10) res = res + Vector.range(12500000+ix).sum() return res }""" ) ) computation2 = self.gateway.requestComputation( makeComputationDefinitionFromIVCs( expr2, ForaNative.makeSymbol("Call") ) ) try: response = self.gateway.finalResponses.get(timeout=60.0) except Queue.Empty: response = None self.assertTrue(response is not None) self.assertTrue(response[1].isResult()) self.gateway.deprioritizeComputation(computation2)
def test_cfgSplitting_1(self): cfg1 = self.parseStringToFunction("fun(f) { f(1) + f(1+3) }").toCFG(1) steps = ForaNative.extractApplyStepsFromControlFlowGraph( cfg1, "block_0Let") self.assertEqual(len(steps), 4) splits = ForaNative.splitControlFlowGraph(cfg1, "block_0Let") self.assertTrue(splits is not None)
def test_simulateGetItem(self): n = 42 ivc = FORA.eval("object { m: %s }" % n).implVal_ res = ForaNative.simulateApply( ForaNative.ImplValContainer( (ivc, self.Symbol_member, makeSymbolIvc("m")))) self.assertEqual(res, ForaNative.ImplValContainer(n))
def test_TypedFora_TupleGetItem(self): tupJovs = [ ForaNative.parseStringToJOV("({String}, {String})"), ForaNative.parseStringToJOV("({String}, *)"), ForaNative.parseStringToJOV("({String}, 'a string 2')"), ForaNative.parseStringToJOV("('a string', 'a string 2')"), ForaNative.parseStringToJOV("('a string', *)"), ForaNative.parseStringToJOV("('a string', {String})"), ForaNative.parseStringToJOV("(*, 'a string 2')"), ForaNative.parseStringToJOV("(*, *)"), ForaNative.parseStringToJOV("(*, {String})") ] layoutStyles = [ TypedFora.RefcountStyle.Pooled(), TypedFora.RefcountStyle.AsValueUnowned(), TypedFora.RefcountStyle.AsValueOwned() ] instance = ForaNative.ImplValContainer( ("a string", "a string 2") ) for tupJov in tupJovs: for refcountStyle in layoutStyles: for index in range(2): type = TypedFora.Type(tupJov, refcountStyle) callable = self.generateTupleGetitemCallable(type, index) def validator(result): return result == instance[index] self.runSimpleEvaluation(callable, [instance], validator)
def wireWorkerToClient_(self, ix1, ix2): workerChannel1, clientChannel1 = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) workerChannel2, clientChannel2 = StringChannelNative.InMemoryStringChannel( self.callbackScheduler) self.workersVdmsAndEventHandlers[ix1][0].addCumulusClient( clientId(ix2), [workerChannel1, workerChannel2], ForaNative.ImplValContainer(), self.callbackScheduler) self.clientsAndVdms[ix2][0].addMachine( self.machineIds[ix1], [clientChannel1, clientChannel2], ForaNative.ImplValContainer(), self.callbackScheduler)
def test_covers(self): vdm = FORANative.VectorDataManager( callbackScheduler, Setup.config().maxPageSizeInBytes ) context = ExecutionContext.ExecutionContext(dataManager = vdm) randomJOVGenerator = FORANative.RandomJOVGenerator(0, context) self.treeElementCoveringTest(coverageTree, [], [], randomJOVGenerator)
def _testJOVTextSet(self, jovStrings, jovMappings): jovts = [ForaNative.parseStringToJOVT(x) for x in jovStrings] tree = ForaNative.JudgmentOnValueTree(jovts) for jmtString, targetIx in jovMappings.iteritems(): testJovt = ForaNative.parseStringToJOVT(jmtString) searchedIx = tree.searchForJOVT(testJovt) self.assertEqual( targetIx, searchedIx, "JOV Tree:\n%s\nError: %s mapped to %s instead of %s" % (str(tree), str(testJovt), searchedIx, targetIx))
def runSimpleEvaluation(self, inputType, outputType): mainRuntime = Runtime.getMainRuntime() foraCompiler = mainRuntime.getTypedForaCompiler() while foraCompiler.anyCompilingOrPending(): time.sleep(.01) aParticularStringValue = ForaNative.ImplValContainer(aBigString) callable = self.generateSimpleCallable(inputType, outputType) jumpTarget = foraCompiler.compile(callable) import gc gc.collect() for passIndex in range(PASS_COUNT): #type values are memoized, so we can't assume that the value has a refcount # of exactly one totalStringCount = ForaNative.totalStringCount() totalImplvalCount = ForaNative.totalImplvalCount() anExecutionContext = ExecutionContext.ExecutionContext( dataManager = VectorDataManager.constructVDM(self.callbackScheduler) ) anExecutionContext.evaluateFunctionPointer(jumpTarget, aParticularStringValue) self.assertTrue(anExecutionContext.isFinished()) res = anExecutionContext.getFinishedResult() self.assertTrue(not res.isException()) self.assertEqual(res.asResult.result, aParticularStringValue) anExecutionContext.teardown() res = None #verify final refcounts self.assertEqual( aParticularStringValue.getStringObjectRefcount(), 1, "refcounts weren't maintained in %s->%s. %s != 1" % ( inputType, outputType, aParticularStringValue.getStringObjectRefcount() ) ) self.assertEqual( (totalStringCount, totalImplvalCount), (ForaNative.totalStringCount(), ForaNative.totalImplvalCount()), "refcounts weren't maintained in " + str(inputType) + "->" + str(outputType) )
def parseStringToPythonAst(text): lineOffsets = computeLineOffsets(text) try: if text not in astCache_: astCache_[text] = convertPythonAstToForaPythonAst( ast.parse(text), lineOffsets) return astCache_[text] except SyntaxError as e: return ForaNative.PythonParseError(e.msg, e.filename, e.lineno, e.offset, e.text) except TypeError as e: return ForaNative.PythonParseError(str(e.message))
def test_serialResults_3(self): funString = "fun(f, x) { let res = f((x, *x)); res + 1 }" cfg = self.parseStringToFunction(funString).toCFG(2) fImplval = FORA.extractImplValContainer( FORA.eval("fun(x) { size(x) }")) xImplVal = ForaNative.ImplValContainer((1, 2, 3)) nodeValues = (fImplval, xImplVal) serialResult = self.serialResult(cfg, nodeValues, "block_5Let") self.assertEqual(serialResult.asResult.result.asResult.result, ForaNative.ImplValContainer(5))
def calculateTime(count): t0 = time.time() response = self.evaluateWithGateway( makeComputationDefinitionFromIVCs( expr, ForaNative.makeSymbol("Call"), ForaNative.ImplValContainer(count))) self.assertTrue(response[1].isResult()) stats = response[2] return (stats.timeSpentInInterpreter, stats.timeSpentInCompiler, time.time() - t0)
def test_resolveAxiomDirectly_smallStrings(self): instance = ForaNative.ImplValContainer( ("s1", ForaNative.makeSymbol("Operator"), ForaNative.makeSymbol("+"), "s2")) jov = ForaNative.implValToJOV(instance) joa = self.axioms.resolveAxiomDirectly(self.compiler, jov.getTuple()) self.assertEqual(len(joa.throwPart()), 0) self.assertEqual(len(joa.resultPart()), 1) result = joa.resultPart()[0] self.assertEqual(result, ForaNative.parseStringToJOV('"s1s2"'))
def instantiateException(self, exceptionTypeName, exceptionArgsAsPyTuple): """Return an IVC representing an exception of type given by name 'exceptionTypeName'. Returns None if 'exceptionTypeName' isn't a singleton. """ exceptionTypeInstance = self.convertSingletonByName(exceptionTypeName) if exceptionTypeInstance is None: return None args = (self.pyExceptionClass, ForaNative.makeSymbol("CreateInstance"), exceptionTypeInstance, exceptionArgsAsPyTuple) return ForaNative.simulateApply(ForaNative.ImplValContainer(args))
def test_resolveAxiomDirectly_VeryLongComputation(self): vectorIVC = FORA.extractImplValContainer(FORA.eval("[]")) jov = ForaNative.parseStringToJOV(("({Vector([])}, `append, 2)")) joa = self.axioms.resolveAxiomDirectly(self.compiler, jov.getTuple()) self.assertEqual(len(joa.throwPart()),0) self.assertEqual(len(joa.resultPart()),1) result = joa.resultPart()[0] self.assertEqual(result, ForaNative.parseStringToJOV("{Vector([{Int64}])}"))
def test_resolveAxiomDirectly_smallStrings(self): instance = ForaNative.ImplValContainer( ("s1", ForaNative.makeSymbol("Operator"), ForaNative.makeSymbol("+"), "s2") ) jov = ForaNative.implValToJOV(instance) joa = self.axioms.resolveAxiomDirectly(self.compiler, jov.getTuple()) self.assertEqual(len(joa.throwPart()),0) self.assertEqual(len(joa.resultPart()),1) result = joa.resultPart()[0] self.assertEqual(result, ForaNative.parseStringToJOV('"s1s2"'))
def runSimpleEvaluation(self, inputType, outputType): mainRuntime = Runtime.getMainRuntime() foraCompiler = mainRuntime.getTypedForaCompiler() while foraCompiler.anyCompilingOrPending(): time.sleep(.01) aParticularStringValue = ForaNative.ImplValContainer(aBigString) callable = self.generateSimpleCallable(inputType, outputType) jumpTarget = foraCompiler.compile(callable) import gc gc.collect() for passIndex in range(PASS_COUNT): #type values are memoized, so we can't assume that the value has a refcount # of exactly one totalStringCount = ForaNative.totalStringCount() totalImplvalCount = ForaNative.totalImplvalCount() anExecutionContext = ExecutionContext.ExecutionContext( dataManager=VectorDataManager.constructVDM( self.callbackScheduler)) anExecutionContext.evaluateFunctionPointer(jumpTarget, aParticularStringValue) self.assertTrue(anExecutionContext.isFinished()) res = anExecutionContext.getFinishedResult() self.assertTrue(not res.isException()) self.assertEqual(res.asResult.result, aParticularStringValue) anExecutionContext.teardown() res = None #verify final refcounts self.assertEqual( aParticularStringValue.getStringObjectRefcount(), 1, "refcounts weren't maintained in %s->%s. %s != 1" % (inputType, outputType, aParticularStringValue.getStringObjectRefcount())) self.assertEqual((totalStringCount, totalImplvalCount), (ForaNative.totalStringCount(), ForaNative.totalImplvalCount()), "refcounts weren't maintained in " + str(inputType) + "->" + str(outputType))
def _testJOVTextSet(self, jovStrings, jovMappings): jovts = [ForaNative.parseStringToJOVT(x) for x in jovStrings] tree = ForaNative.JudgmentOnValueTree(jovts) for jmtString, targetIx in jovMappings.iteritems(): testJovt = ForaNative.parseStringToJOVT(jmtString) searchedIx = tree.searchForJOVT(testJovt) self.assertEqual(targetIx, searchedIx, "JOV Tree:\n%s\nError: %s mapped to %s instead of %s" % ( str(tree), str(testJovt), searchedIx, targetIx ))
def testMatchStructureCantMatchTooSmallTuple(self): runtime = Runtime.getMainRuntime() axioms = runtime.getAxioms() jov = FORANative.parseStringToJOV( "((1,2,3),`StructureMatch,((nothing, false), (nothing, false), (nothing, false), (nothing, false)))" ) axiom = axioms.getAxiomByJOVT(runtime.getTypedForaCompiler(), jov.asTuple.jov) self.assertTrue( list(axiom.asNative.resultSignature.resultPart().vals) == [FORANative.parseStringToJOV("nothing")])
def test_teardown_simple_2(self): vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes) context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInternalSplitting=False) context.placeInEvaluationState( FORANative.ImplValContainer((FORA.extractImplValContainer( FORA.eval( "fun(){ let f = fun() { throw 1 }; try { f() } catch(...) { throw 2 } }" )), FORANative.symbol_Call))) context.compute() self.assertTrue(context.getFinishedResult().isException())
def test_verifySmallAllocNotRemapping(self): vdmm = ForaNative.createTestingVDMM() ForaNative.executeBigVectorHandleAllocationsForTimePeriod( vdmm, .25, 1024 * 1024) bytes1 = vdmm.totalBytesMmappedCumulatively() ForaNative.executeBigVectorHandleAllocationsForTimePeriod( vdmm, 1.0, 1024 * 1024) bytes2 = vdmm.totalBytesMmappedCumulatively() self.assertTrue(bytes2 <= 1024 * 1024 * 4)
def loadEntireFileDataset(datasetDescriptor, vdid, vdm): try: vectorSlices = produceVDIDSlicesForFile( datasetDescriptor.asEntireFileDataset.file) vector = ForaNative.createFORAFreeBinaryVectorFromSlices( vectorSlices, vdm) if not vdm.loadImplvalIntoUnloadedVectorHandle(vdid, vector): raise DatasetLoadException("Couldn't load dataset into VDM") except InvalidDatasetException as e: logging.error("Failed to load dataset: %s", e.message) if not vdm.loadImplvalIntoUnloadedVectorHandle( vdid, ForaNative.ImplValContainer(e.message)): raise DatasetLoadException("Couldn't load dataset into VDM")
def test_resolveAxiomDirectly_VeryLongComputation(self): vectorIVC = FORA.extractImplValContainer(FORA.eval("[]")) jov = ForaNative.parseStringToJOV(("({Vector([])}, `append, 2)")) joa = self.axioms.resolveAxiomDirectly(self.compiler, jov.getTuple()) self.assertEqual(len(joa.throwPart()), 0) self.assertEqual(len(joa.resultPart()), 1) result = joa.resultPart()[0] self.assertEqual(result, ForaNative.parseStringToJOV("{Vector([{Int64}])}"))
def extractPagedUnloadedVector(self, vdm, count): context = ExecutionContext.ExecutionContext( dataManager=vdm, allowInternalSplitting=False) evaluate( context, FORA.extractImplValContainer( FORA.eval("fun() { Vector.range(%s).paged }" % count)), FORANative.ImplValContainer(FORANative.makeSymbol("Call"))) pagedVec = context.getFinishedResult().asResult.result vdm.unloadAllPossible() return pagedVec
def __init__(self, x=None): object.__init__(self) if x is None: self.implVal_ = ForaNative.nothing elif isinstance(x, FORAValue): self.implVal_ = x.implVal_ elif isinstance(x, ForaNative.ImplValContainer): self.implVal_ = x elif isinstance(x, tuple): self.implVal_ = ForaNative.ImplValContainer( tuple([FORAValue(elt).implVal_ for elt in x])) else: #this will attempt to convert numerics #and will otherwise represent the value as a python object self.implVal_ = ForaNative.ImplValContainer(x)
def loadEntireS3Dataset(datasetDescriptor, s3InterfaceFactory, vdid, vdm): try: vectorSlices = produceVDIDSlicesForEntireS3Dataset( s3InterfaceFactory, datasetDescriptor.asEntireS3Dataset.dataset, vdm) vector = ForaNative.createFORAFreeBinaryVectorFromSlices( vectorSlices, vdm) if not vdm.loadImplvalIntoUnloadedVectorHandle(vdid, vector): raise DatasetLoadException("Couldn't load dataset into VDM") except InvalidDatasetException as e: if not vdm.loadImplvalIntoUnloadedVectorHandle( vdid, ForaNative.ImplValContainer(e.message)): raise DatasetLoadException("Couldn't load dataset into VDM")
def submitComputationOnClient(self, clientIndex, expressionText, **freeVariables): if isinstance(expressionText, ForaNative.ImplValContainer): computationDefinition = CumulusNative.ComputationDefinition.Root( CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm( [CumulusNative.ComputationDefinitionTerm.Value(x, None) for x in expressionText.getTuple()] ) ) elif isinstance(expressionText, CumulusNative.ComputationDefinition): computationDefinition = expressionText else: varNames = list(freeVariables.keys()) expr = FORA.eval("fun(" + ",".join(varNames) + ") {" + expressionText + " } ") computationDefinition = createComputationDefinition( FORA.extractImplValContainer( expr ), ForaNative.makeSymbol("Call"), *[freeVariables[v] for v in varNames] ) computationId = self.getClient(clientIndex).createComputation(computationDefinition) self.getClient(clientIndex).setComputationPriority( computationId, CumulusNative.ComputationPriority(1) ) return computationId
def runTest(): results.append( ForaNative.executeMemoryUpdatesForTimePeriod( measurementTime, blockSize ) )
def makeComputationDefinitionFromIVCs(*args): return CumulusNative.ComputationDefinition.Root( CumulusNative.ImmutableTreeVectorOfComputationDefinitionTerm( [CumulusNative.ComputationDefinitionTerm.Value(ForaNative.ImplValContainer(x), None) for x in args] ) )
def test_sortManySmallVectors(self): self.desirePublisher.desireNumberOfWorkers(4, blocking=True) expr = FORA.extractImplValContainer( FORA.eval( """fun() { let shouldAllBeTrue = Vector.range(20, fun(o) { sorting.isSorted( sort(Vector.range(50000 + o, fun(x) { x / 10 })) ) }); for s in shouldAllBeTrue { if (not s) return false } return true }""" ) ) response = self.evaluateWithGateway( makeComputationDefinitionFromIVCs( expr, ForaNative.makeSymbol("Call") ) ) self.assertTrue(response[1].isResult()) self.assertTrue(response[1].asResult.result.pyval == True, response[1].asResult.result.pyval)
def test_expensive_brownian(self): self.desirePublisher.desireNumberOfWorkers(2, blocking=True) expr = FORA.extractImplValContainer( FORA.eval( """fun() { let brownian = fun(x,t) { if (t == 0) return sum(0.0, x * 10.0**7.0) else { let (l,r) = cached(brownian(x - 1, t - 1), brownian(x + 1, t - 1)); return sum(0.0, x * 10.0**7.0) + l + r } } brownian(0, 5) }""" ) ) response = self.evaluateWithGateway( makeComputationDefinitionFromIVCs( expr, ForaNative.makeSymbol("Call") ) ) self.assertTrue(response[1].isResult())
def test_manyDuplicateCachecallsAndAdding(self): self.desirePublisher.desireNumberOfWorkers(2, blocking=True) expr = FORA.extractImplValContainer( FORA.eval( """fun() { Vector.range(20).papply(fun(x) { x + cached(sum(0,10**11))[0] }) }""" ) ) computationId = self.gateway.requestComputation( makeComputationDefinitionFromIVCs( expr, ForaNative.makeSymbol("Call") ) ) time.sleep(5) self.desirePublisher.desireNumberOfWorkers(4, blocking=True) try: response = self.gateway.finalResponses.get(timeout=240.0) except Queue.Empty: response = None self.assertTrue(response is not None) self.assertTrue(response[1].isResult()) self.gateway.deprioritizeComputation(computationId)
def test_sortALargeVectorWithFourWorkers(self): self.desirePublisher.desireNumberOfWorkers(4, blocking=True) expr = FORA.extractImplValContainer( FORA.eval( """fun() { let v = Vector.range(12500000, {_%3.1415926}); if (sorting.isSorted(sorting.sort(v))) 'sorted' }""" ) ) try: response = self.evaluateWithGateway( makeComputationDefinitionFromIVCs( expr, ForaNative.makeSymbol("Call") ), timeout=120.0 ) except Queue.Empty: response = None if response is None: try: dumpFun = self.dumpSchedulerEventStreams dumpFun() except: logging.warn("Wanted to dump CumulusWorkerEvents, but couldn't"); self.assertTrue(response is not None) self.assertTrue(response[1].isResult()) self.assertTrue(response[1].asResult.result.pyval == 'sorted', response[1].asResult.result.pyval)
def measureMmapPerformance(self, measurementTime, blockSize, threadCount, initializeData): t0 = time.time() results = [] vdmm = ForaNative.createTestingVDMM() def runTest(): results.append( ForaNative.executeMMapAndFreeForTimePeriod( vdmm, measurementTime, blockSize, initializeData ) ) threads = [threading.Thread(target=runTest) for ix in range(threadCount)] for t in threads: t.start() for t in threads: t.join() return (time.time() - t0) / (sum(results) / 1024.0 ** 3)
def produceVDIDSlicesForSingleBucketKeyPair(s3Interface, bucketname, keyname, s3Dataset): try: totalBytes = s3Interface.getKeySize(bucketname, keyname) chunks = getAppropriateChunksForSize(totalBytes, CHUNK_SIZE) slices = [] for lowIndex, highIndex in chunks: externalDatasetDesc = ForaNative.ExternalDatasetDescriptor.S3DatasetSlice( s3DatasetWithKeyname(s3Dataset, keyname), lowIndex, highIndex ) vectorDataId = ForaNative.VectorDataID.External(externalDatasetDesc) vectorDataIdSlice = ForaNative.createVectorDataIDSlice(vectorDataId, 0, highIndex - lowIndex) slices.append(vectorDataIdSlice) return slices except S3Interface.S3InterfaceError as e: message = "Error loading S3 dataset: %s/%s:\n%s" % ( bucketname, keyname, e ) logging.error(message) raise InvalidDatasetException(message)
def reasonAboutExpression(self, expression, **variableJudgments): reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.axioms, False) keys = sorted(list(variableJudgments.keys())) functionText = "fun(" + ",".join(['_'] + keys) + ") { " + expression + " }" frame = reasoner.reason( makeJovt( FORANative.parseStringToJOV(functionText), *[FORANative.parseStringToJOV(variableJudgments[k]) for k in keys] ) ) self.dumpReasonerSummary(reasoner, frame) return frame
def convertClassInstanceDescription(self, objectId, classInstanceDescription, convertedValues): classMemberNameToImplVal = { classMemberName: convertedValues[memberId] for classMemberName, memberId in classInstanceDescription.classMemberNameToClassMemberId.iteritems() } classImplVal = convertedValues[classInstanceDescription.classId] if classImplVal.isSymbol(): convertedValues[objectId] = classImplVal return memberNames = tuple(sorted(name for name in classMemberNameToImplVal.iterkeys())) memberValues = tuple(classMemberNameToImplVal[name] for name in memberNames) convertedValueOrNone = ForaNative.simulateApply( ForaNative.ImplValContainer( (classImplVal, Symbol_CreateInstance, ForaNative.CreateNamedTuple(memberValues, memberNames)) ) ) if convertedValueOrNone is None: raise pyfora.PythonToForaConversionError( ("An internal error occurred: " + "function stage 1 simulation unexpectedly returned None") ) convertedValues[objectId] = convertedValueOrNone
def __init__(self, nativeConstantConverter, nativeDictConverter, nativeTupleConverter, nativeListConverter, vdmOverride, builtinMemberMapping, purePythonModuleImplVal): self.boundExpressions = {} self.constantConverter = ConstantConverter.ConstantConverter( nativeConstantConverter=nativeConstantConverter ) self.nativeDictConverter = nativeDictConverter self.nativeTupleConverter = nativeTupleConverter self.nativeListConverter = nativeListConverter self.vdm_ = vdmOverride self.nativeConverter = ForaNative.makePythonAstConverter( nativeConstantConverter, nativeListConverter, nativeTupleConverter, nativeDictConverter, purePythonModuleImplVal, builtinMemberMapping )
def test_cfgSplitting_7(self): funString = "fun(f) { f(1) + f(2) }" cfg = self.parseStringToFunction(funString).toCFG(1) splits = ForaNative.splitControlFlowGraph(cfg, "block_0Let") self.assertTrue(splits is not None)
def assertFrameHasResultJOV(self, frame, resultJOV): if isinstance(resultJOV, str): resultJOV = FORANative.parseStringToJOV(resultJOV) self.assertTrue( len(frame.exits().resultPart().vals) == 1, frame.exits()) self.assertEqual(frame.exits().resultPart()[0], resultJOV)
def loadAxiomSignaturesFromFile(self, axioms_file): """ Loads axiom signatures from a file `axioms_file`. This file should contain axiom signatures on each line, but also can have commented lines (starting with a `#`) or commented blocks (lines surrounded by lines of `\"""`), as in python syntax. """ axiom_signatures = [] line_number = 0 in_comment = False assert os.path.exists(axioms_file), ( "unable to open file `%s'\n" %axioms_file ) with open(axioms_file) as f: for line in f: line_number += 1 if re.search('^"""', line): in_comment = (in_comment != True) # in_comment = in_comment XOR True continue if not in_comment and not re.search('^#', line) and len(line.strip()) > 0: try: axiom_signatures.append(FORANative.parseStringToJOV( line.strip()).getTuple()) except Exception as inst: logging.warn("unable to parse to JOV: `%s', %s:%s", line.strip(), axioms_file, line_number) raise inst return axiom_signatures
def test_vecWithinVec(self): self.desirePublisher.desireNumberOfWorkers(4, blocking=True) expr = FORA.extractImplValContainer( FORA.eval( """fun() { let v = Vector.range(10) ~~ fun(x) { Vector.range(1000000 + x).paged }; v = v.paged; let res = () for elt in v res = res + (elt,) res..apply(fun(tupElt) { tupElt.sum() }) 'got to the end' }""" ) ) response = self.evaluateWithGateway( makeComputationDefinitionFromIVCs( expr, ForaNative.makeSymbol("Call") ) ) self.assertTrue(response[1].isResult()) self.assertTrue(response[1].asResult.result.pyval == 'got to the end', response[1].asResult.result.pyval)
def sim(memberName): return ForaNative.simulateApply( ForaNative.ImplValContainer( (ivc, makeSymbolIvc("Member"), makeSymbolIvc(memberName)) ) ).pyval