Пример #1
0
def initialize(setupObjectToUse=None, useLocalEvaluator=True):
    global _builtin
    if _builtin is not None:
        return

    Runtime.initialize(setupObjectToUse)
    ModuleImporter.initialize(setupObjectToUse)
    Evaluator.initialize(setupObjectToUse, useLocalEvaluator)

    _builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
Пример #2
0
def main(parsedArguments):
    Runtime.initialize()

    testMissingMapFile()
    testMissingDatFiles()
    testMissingIndexFiles()

    testCorruptMapFile()
    testCorruptDatFiles()
    testCorruptIndexFiles()
Пример #3
0
def main(parsedArguments):
    Runtime.initialize()

    testMissingMapFile()
    testMissingDatFiles()
    testMissingIndexFiles()

    testCorruptMapFile()
    testCorruptDatFiles()
    testCorruptIndexFiles()
Пример #4
0
def initialize(setupObjectToUse = None, useLocalEvaluator = True):
    global _builtin
    if _builtin is not None:
        return

    Runtime.initialize(setupObjectToUse)
    ModuleImporter.initialize(setupObjectToUse)
    Evaluator.initialize(setupObjectToUse, useLocalEvaluator)

    _builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
Пример #5
0
def main(argv):
    t0 = time.time()

    Runtime.initialize(Setup.defaultSetup())

    runtime = Runtime.getMainRuntime()
    handler = runtime.getInterpreterTraceHandler()

    handler.replayTracesFromFile(argv[1])

    print "took ", time.time() - t0
def main(argv):
	t0 = time.time()

	Runtime.initialize(Setup.defaultSetup())
	
	runtime = Runtime.getMainRuntime()
	handler = runtime.getInterpreterTraceHandler()

	handler.replayTracesFromFile(argv[1])

	print "took ", time.time() - t0
Пример #7
0
def main(parsedArguments):
    Runtime.initialize()

    tester = AxiomTester(
        parsedArguments.numRandVals,
        parsedArguments.numRelaxations,
        parsedArguments.maxForRelax,
        parsedArguments.maxForRand,
        parsedArguments.testAxiomsPath,
        parsedArguments.seed
        )

    tester.test_axiom_consistency()
Пример #8
0
    def test_extractPausedComputationDuringVectorLoad(self):
        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(
            dataManager = vdm,
            allowInterpreterTracing = False
            )

        context.evaluate(
            FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
            FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
            )

        pagedVec = context.getFinishedResult().asResult.result

        context.placeInEvaluationState(
            FORANative.ImplValContainer(
                (pagedVec,
                FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
                FORANative.ImplValContainer(0))
                )
            )

        vdm.unloadAllPossible()

        context.resume()

        self.assertTrue(context.isVectorLoad())

        computation = context.extractPausedComputation()

        self.assertEqual(len(computation.frames),1)
Пример #9
0
 def setUp(self):
     self.callbackScheduler = CallbackScheduler.singletonForTesting()
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.compiler = self.runtime.getTypedForaCompiler()
     self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(
         FORA.builtin().implVal_)
Пример #10
0
    def __init__(self, numRandVals, numRelaxations, maxForRelax, maxForRand, testAxiomsPath, seed):
        object.__init__(self)
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.callbackSchedulerFactory = self.callbackScheduler.getFactory()
        
        self.numRandVals = numRandVals
        self.numRelaxations = numRelaxations
        self.maxForRelax = maxForRelax
        self.maxForRand = maxForRand
        self.seed = seed

        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.typed_fora_compiler = self.runtime.getTypedForaCompiler()

        if testAxiomsPath is not None:
            pathToUse = testAxiomsPath
        else:
            pathToUse = UNIT_TEST_AXIOMS_PATH

        self.axiom_signatures_to_test = self.loadAxiomSignaturesFromFile(pathToUse)

        self.axiom_groups = []
        for i in range(self.axioms.axiomCount):
            self.axiom_groups.append(self.axioms.getAxiomGroupByIndex(i))

        self.symbol_strings = self.loadSymbolStrings()

        numpy.random.seed(seed)
Пример #11
0
    def test_extractPausedComputationDuringVectorLoad(self):
        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler,
                                           Setup.config().maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(
            dataManager=vdm, allowInterpreterTracing=False)

        context.evaluate(
            FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
            FORANative.ImplValContainer(FORANative.makeSymbol("Call")))

        pagedVec = context.getFinishedResult().asResult.result

        context.placeInEvaluationState(
            FORANative.ImplValContainer(
                (pagedVec,
                 FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
                 FORANative.ImplValContainer(0))))

        vdm.unloadAllPossible()

        context.resume()

        self.assertTrue(context.isVectorLoad())

        computation = context.extractPausedComputation()

        self.assertEqual(len(computation.frames), 1)
Пример #12
0
    def test_resumePausedComputationWithResult(self):
        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler,
                                           Setup.config().maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(
            dataManager=vdm,
            allowInterpreterTracing=False,
            allowInternalSplitting=False)

        text = """
        let f = fun(v, ix) {
            if (ix > 0)
                {
                let (v2,res) = f(v,ix-1);
                return (v2, res + v2[0])
                }

            `TriggerInterruptForTesting()

            return (v, 0)
            };

        f([1], 10)
        """

        evaluate(
            context,
            FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")),
            FORANative.ImplValContainer(FORANative.makeSymbol("Call")))

        assert context.isInterrupted()

        pausedComp = context.extractPausedComputation()

        framesToUse = pausedComp.asThread.computation.frames[0:5]

        pausedComp2 = FORANative.PausedComputationTree(
            FORANative.PausedComputation(
                framesToUse,
                FORA.extractImplValContainer(
                    FORA.eval("([2], 0)", keepAsForaValue=True)), False))

        context.resumePausedComputation(pausedComp2)

        context.copyValuesOutOfVectorPages()
        context.pageLargeVectorHandles(0)

        context.resetInterruptState()
        context.compute()

        self.assertTrue(context.isFinished())

        result = context.getFinishedResult()

        self.assertTrue(result.asResult.result[1].pyval == 6)
    def testMatchStructureCantMatchTooSmallTuple(self):
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()

        jov = FORANative.parseStringToJOV("((1,2,3),`StructureMatch,((nothing, false), (nothing, false), (nothing, false), (nothing, false)))")

        axiom = axioms.getAxiomByJOVT(runtime.getTypedForaCompiler(), jov.asTuple.jov)

        self.assertTrue(list(axiom.asNative.resultSignature.resultPart().vals) == [FORANative.parseStringToJOV("nothing")])
Пример #14
0
    def test_resumePausedComputationWithResult(self):
        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(
            dataManager = vdm,
            allowInterpreterTracing = False
            )

        text = """
        let f = fun(v, ix) {
            if (ix > 0)
                {
                let (v2,res) = f(v,ix-1);
                return (v2, res + v2[0])
                }

            `TriggerInterruptForTesting()

            return (v, 0)
            };

        f([1], 10)
        """

        context.evaluate(
            FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")),
            FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
            )

        assert context.isInterrupted()

        pausedComp = context.extractPausedComputation()

        framesToUse = pausedComp.frames[0:5]

        pausedComp2 = FORANative.PausedComputation(
            framesToUse,
            FORA.extractImplValContainer(FORA.eval("([2], 0)", keepAsForaValue=True)),
            False
            )

        context.resumePausedComputation(pausedComp2)

        context.copyValuesOutOfVectorPages()
        context.pageLargeVectorHandles(0)

        context.resetInterruptState()
        context.resume()

        self.assertTrue( context.isFinished() )

        result = context.getFinishedResult()

        self.assertTrue(result.asResult.result[1].pyval == 6)
Пример #15
0
    def testAxiomLookups(self):
        axioms = Runtime.getMainRuntime().getAxioms()

        jmtsToTest = [axiom.signature() for axiom in axioms]

        for tupJmt in jmtsToTest:
            self.assertEqual(
                axioms.axiomSearchLinear(tupJmt),
                axioms.axiomSearchTree(tupJmt),
                "Didn't produce same results for %s" % str(tupJmt))
Пример #16
0
    def testAxiomLookups(self):
        axioms = Runtime.getMainRuntime().getAxioms()

        jmtsToTest = [axiom.signature() for axiom in axioms]

        for tupJmt in jmtsToTest:
            self.assertEqual(
                axioms.axiomSearchLinear(tupJmt),
                axioms.axiomSearchTree(tupJmt),
                "Didn't produce same results for %s" % str(tupJmt)
                )
Пример #17
0
    def triggerCompilation(self, *args):
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler = runtime.getTypedForaCompiler()
        instructionGraph = runtime.getInstructionGraph()
        
        reasoner = ForaNative.SimpleForwardReasoner(compiler, instructionGraph, axioms)
        
        frame = reasoner.reasonAboutComputationDefinition(self.cumulusComputationDefinition)

        logging.critical("Result was %s", frame.exits())
Пример #18
0
    def runSimpleEvaluation(self, inputType, outputType):
        mainRuntime = Runtime.getMainRuntime()
        foraCompiler = mainRuntime.getTypedForaCompiler()

        while foraCompiler.anyCompilingOrPending():
            time.sleep(.01)

        aParticularStringValue = ForaNative.ImplValContainer(aBigString)

        callable = self.generateSimpleCallable(inputType, outputType)

        jumpTarget = foraCompiler.compile(callable)

        import gc
        gc.collect()

        for passIndex in range(PASS_COUNT):
            #type values are memoized, so we can't assume that the value has a refcount
            # of exactly one
            totalStringCount = ForaNative.totalStringCount()
            totalImplvalCount = ForaNative.totalImplvalCount()

            anExecutionContext = ExecutionContext.ExecutionContext(
                dataManager = VectorDataManager.constructVDM(self.callbackScheduler)
                )

            anExecutionContext.evaluateFunctionPointer(jumpTarget, aParticularStringValue)

            self.assertTrue(anExecutionContext.isFinished())

            res = anExecutionContext.getFinishedResult()

            self.assertTrue(not res.isException())
            self.assertEqual(res.asResult.result, aParticularStringValue)

            anExecutionContext.teardown()

            res = None

            #verify final refcounts
            self.assertEqual(
                aParticularStringValue.getStringObjectRefcount(),
                1,
                "refcounts weren't maintained in %s->%s. %s != 1" % (
                    inputType,
                    outputType,
                    aParticularStringValue.getStringObjectRefcount()
                    )
                )
            self.assertEqual(
                (totalStringCount, totalImplvalCount),
                (ForaNative.totalStringCount(), ForaNative.totalImplvalCount()),
                "refcounts weren't maintained in " + str(inputType) + "->" + str(outputType)
                )
Пример #19
0
    def setUp(self):
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.compiler = self.runtime.getTypedForaCompiler()
        self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)

        pyforaPath = os.path.join(os.path.split(pyfora.__file__)[0], "fora/purePython")
        self.purePythonAsJOV = FORANative.JudgmentOnValue.Constant(FORA.importModule(pyforaPath).implVal_)
        
        self.instructionGraph = self.runtime.getInstructionGraph()
        self.reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.instructionGraph, self.axioms)
Пример #20
0
    def setUp(self):
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.compiler = self.runtime.getTypedForaCompiler()
        self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)

        pyforaPath = os.path.join(os.path.split(pyfora.__file__)[0], "fora/purePython")
        self.purePythonAsJOV = FORANative.JudgmentOnValue.Constant(FORA.importModule(pyforaPath).implVal_)

        self.instructionGraph = self.runtime.getInstructionGraph()
        self.reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.instructionGraph, self.axioms)
Пример #21
0
def reasonAboutForaCode(module, testName):
    if reasoner[0] is None:
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler = runtime.getTypedForaCompiler()
        reasoner[0] = ForaNative.SimpleForwardReasoner(compiler, axioms, False)

    moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)
    frame = reasoner[0].reason(makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))

    dumpReasonerSummary(reasoner[0], frame)

    return True
Пример #22
0
    def triggerCompilation(self, *args):
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler = runtime.getTypedForaCompiler()
        instructionGraph = runtime.getInstructionGraph()

        reasoner = ForaNative.SimpleForwardReasoner(compiler, instructionGraph,
                                                    axioms)

        frame = reasoner.reasonAboutComputationDefinition(
            self.cumulusComputationDefinition)

        logging.critical("Result was %s", frame.exits())
Пример #23
0
    def runSimpleEvaluation(self, inputType, outputType):
        mainRuntime = Runtime.getMainRuntime()
        foraCompiler = mainRuntime.getTypedForaCompiler()

        while foraCompiler.anyCompilingOrPending():
            time.sleep(.01)

        aParticularStringValue = ForaNative.ImplValContainer(aBigString)

        callable = self.generateSimpleCallable(inputType, outputType)

        jumpTarget = foraCompiler.compile(callable)

        import gc
        gc.collect()

        for passIndex in range(PASS_COUNT):
            #type values are memoized, so we can't assume that the value has a refcount
            # of exactly one
            totalStringCount = ForaNative.totalStringCount()
            totalImplvalCount = ForaNative.totalImplvalCount()

            anExecutionContext = ExecutionContext.ExecutionContext(
                dataManager=VectorDataManager.constructVDM(
                    self.callbackScheduler))

            anExecutionContext.evaluateFunctionPointer(jumpTarget,
                                                       aParticularStringValue)

            self.assertTrue(anExecutionContext.isFinished())

            res = anExecutionContext.getFinishedResult()

            self.assertTrue(not res.isException())
            self.assertEqual(res.asResult.result, aParticularStringValue)

            anExecutionContext.teardown()

            res = None

            #verify final refcounts
            self.assertEqual(
                aParticularStringValue.getStringObjectRefcount(), 1,
                "refcounts weren't maintained in %s->%s. %s != 1" %
                (inputType, outputType,
                 aParticularStringValue.getStringObjectRefcount()))
            self.assertEqual((totalStringCount, totalImplvalCount),
                             (ForaNative.totalStringCount(),
                              ForaNative.totalImplvalCount()),
                             "refcounts weren't maintained in " +
                             str(inputType) + "->" + str(outputType))
Пример #24
0
def reasonAboutForaCode(module, testName):
    if reasoner[0] is None:
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler = runtime.getTypedForaCompiler()
        reasoner[0] = ForaNative.SimpleForwardReasoner(compiler, axioms, False)

    moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)
    frame = reasoner[0].reason(
        makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))

    dumpReasonerSummary(reasoner[0], frame)

    return True
Пример #25
0
    def setUp(self):
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.native_runtime = self.runtime.getTypedForaCompiler()

        symbol_strings_set = Set()
        self.symbol_strings = []
        for i in range(self.axioms.axiomCount):
            sig = str(self.axioms.getAxiomGroupByIndex(i).signature())
            for j in [l.strip(' ') for l in sig.strip("()").split(',')]:
                if re.search('^`', j):
                    if j not in symbol_strings_set:
                        symbol_strings_set.add(j)
                        self.symbol_strings.append(j.strip('`'))
Пример #26
0
    def testMatchStructureCantMatchTooSmallTuple(self):
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()

        jov = FORANative.parseStringToJOV(
            "((1,2,3),`StructureMatch,((nothing, false), (nothing, false), (nothing, false), (nothing, false)))"
        )

        axiom = axioms.getAxiomByJOVT(runtime.getTypedForaCompiler(),
                                      jov.asTuple.jov)

        self.assertTrue(
            list(axiom.asNative.resultSignature.resultPart().vals) ==
            [FORANative.parseStringToJOV("nothing")])
Пример #27
0
def reasonAboutForaCode(module, moduleName, testName):
    if reasoner[0] is None:
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler[0] = runtime.getTypedForaCompiler()
        instructionGraph = runtime.getInstructionGraph()
        reasoner[0] = ForaNative.SimpleForwardReasoner(compiler[0], instructionGraph, axioms)

    moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)

    t0 = time.time()
    frame = reasoner[0].reasonAboutApply(makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))
    print "reasoning ", moduleName + "." + testName, " took ", time.time() - t0

    dumpReasonerSummary(moduleName, testName, reasoner[0], frame)

    return True
Пример #28
0
    def test_extractStacktrace(self):
        text = """fun() {
            let x = 0;
            let v = [1,2,3]
            while (x < 100000) {
                x = x + 1
                }
            (x,v)
            }"""

        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
        context = ExecutionContext.ExecutionContext(
            dataManager = vdm,
            allowInterpreterTracing = False
            )

        context.interruptAfterCycleCount(1000)

        context.evaluate(
            FORA.extractImplValContainer(FORA.eval(text)),
            FORANative.symbol_Call
            )

        trace = context.extractStacktrace(True)

        self.assertTrue(len(trace) == 1)

        stacktrace = trace[0][0]
        frameData = trace[0][1]

        codeLocation = FORANative.getCodeLocation(stacktrace.getIDs()[0])
        self.assertTrue(codeLocation is not None)


        self.assertTrue(frameData.wasCompiled is False)
        self.assertTrue(len(frameData.inScopeValues) == 2)
        self.assertTrue([str(x) for x in frameData.inScopeValues.keys] == ['x', 'v'])

        values = [frameData.inScopeValues[x] for x in frameData.inScopeValues.keys]
        self.assertTrue(values[0].isInt64())
        self.assertTrue(values[1].isVector())

        context.teardown()
Пример #29
0
    def test_extractStacktrace(self):
        text = """fun() {
            let x = 0;
            let v = [1,2,3]
            while (x < 100000) {
                x = x + 1
                }
            (x,v)
            }"""

        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler,
                                           Setup.config().maxPageSizeInBytes)
        context = ExecutionContext.ExecutionContext(
            dataManager=vdm, allowInterpreterTracing=False)

        context.interruptAfterCycleCount(1000)

        context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
                         FORANative.symbol_Call)

        trace = context.extractStacktrace(True)

        self.assertTrue(len(trace) == 1)

        stacktrace = trace[0][0]
        frameData = trace[0][1]

        codeLocation = FORANative.getCodeLocation(stacktrace.getIDs()[0])
        self.assertTrue(codeLocation is not None)

        self.assertTrue(frameData.wasCompiled is False)
        self.assertTrue(len(frameData.inScopeValues) == 2)
        self.assertTrue([str(x)
                         for x in frameData.inScopeValues.keys] == ['x', 'v'])

        values = [
            frameData.inScopeValues[x] for x in frameData.inScopeValues.keys
        ]
        self.assertTrue(values[0].isInt64())
        self.assertTrue(values[1].isVector())

        context.teardown()
Пример #30
0
    def setUp(self):
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.native_runtime = self.runtime.getTypedForaCompiler()
        self.vals_to_test = self.loadValuesFromFile(os.path.join(os.path.split(__file__)[0],
                                                        "AxiomJOA_test.txt"))

        self.evaluator = LocalEvaluator.LocalEvaluator(
                            lambda vdm: CumulusNative.SimpleOfflineCache(self.callbackScheduler, 1000000000),
                            10000000,
                            maxPageSizeInBytes = 100000
                            )
        self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)

        self.knownModulesAsConstantJOVs = dict()
        self.knownModulesAsConstantJOVs["builtin"] = \
                FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
Пример #31
0
    def setUp(self):
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.native_runtime = self.runtime.getTypedForaCompiler()
        self.vals_to_test = self.loadValuesFromFile(
            os.path.join(os.path.split(__file__)[0], "AxiomJOA_test.txt"))

        self.evaluator = LocalEvaluator.LocalEvaluator(
            lambda vdm: CumulusNative.SimpleOfflineCache(
                self.callbackScheduler, 1000000000),
            10000000,
            maxPageSizeInBytes=100000)
        self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)

        self.knownModulesAsConstantJOVs = dict()
        self.knownModulesAsConstantJOVs["builtin"] = \
                FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
Пример #32
0
    def runSimpleEvaluation(self, callable, arguments, validator):
        mainRuntime = Runtime.getMainRuntime()
        foraCompiler = mainRuntime.getTypedForaCompiler()
        jumpTarget = foraCompiler.compile(callable)

        while foraCompiler.anyCompilingOrPending():
            time.sleep(.01)

        gc.collect()

        for passIndex in range(PASS_COUNT):
            totalStringCount = ForaNative.totalStringCount()
            totalImplvalCount = ForaNative.totalImplvalCount()

            anExecutionContext = ExecutionContext.ExecutionContext(
                dataManager = VectorDataManager.constructVDM(self.callbackScheduler)
                )

            anExecutionContext.evaluateFunctionPointer(jumpTarget, *arguments)

            self.assertTrue(anExecutionContext.isFinished())

            res = anExecutionContext.getFinishedResult()

            self.assertTrue(validator(res.asResult.result),
                "invalid result in " + str(callable) + " with " + str(arguments) +
                ". got " + str(res)
                )

            res = None
            anExecutionContext.teardown()

            curRefs = (ForaNative.totalStringCount(), ForaNative.totalImplvalCount())

            self.assertEqual(
                (totalStringCount, totalImplvalCount),
                curRefs,
                "refcounts weren't maintained in " + str(callable) + " with " + str(arguments) +
                ". %s != %s" % (
                    (totalStringCount, totalImplvalCount),
                    curRefs
                    )
                )
Пример #33
0
    def test_extractPausedComputation(self):
        text = """fun() {
            let x = 0;
            while (x < 100000)
                x = x + 1
            x
            }"""

        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
        context = ExecutionContext.ExecutionContext(
            dataManager = vdm,
            allowInterpreterTracing = False
            )

        context.interruptAfterCycleCount(1010)

        context.evaluate(
            FORA.extractImplValContainer(FORA.eval(text)),
            FORANative.symbol_Call
            )

        computation = context.extractPausedComputation()

        self.assertEqual(len(computation.frames), 2)

        self.assertEqual(computation.frames[1].values[0].pyval, 335)

        context2 = ExecutionContext.ExecutionContext(
            dataManager = vdm,
            allowInterpreterTracing = False
            )

        context2.resumePausedComputation(computation)
        context2.resume()

        self.assertEqual(context2.getFinishedResult().asResult.result.pyval, 100000)

        context.teardown()
        context2.teardown()
Пример #34
0
def reasonAboutForaCode(module, moduleName, testName):
    if reasoner[0] is None:
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler[0] = runtime.getTypedForaCompiler()
        instructionGraph = runtime.getInstructionGraph()
        reasoner[0] = ForaNative.SimpleForwardReasoner(compiler[0],
                                                       instructionGraph,
                                                       axioms)

    moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)

    t0 = time.time()
    frame = reasoner[0].reasonAboutApply(
        makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))
    print "reasoning ", moduleName + "." + testName, " took ", time.time() - t0

    dumpReasonerSummary(moduleName, testName, reasoner[0], frame)

    return True
Пример #35
0
    def test_extractPausedComputation(self):
        text = """fun() {
            let x = 0;
            while (x < 100000)
                x = x + 1
            x
            }"""

        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler,
                                           Setup.config().maxPageSizeInBytes)
        context = ExecutionContext.ExecutionContext(
            dataManager=vdm, allowInterpreterTracing=False)

        context.interruptAfterCycleCount(1010)

        context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
                         FORANative.symbol_Call)

        computation = context.extractPausedComputation()

        self.assertEqual(len(computation.frames), 2)

        self.assertEqual(computation.frames[1].values[0].pyval, 335)

        context2 = ExecutionContext.ExecutionContext(
            dataManager=vdm, allowInterpreterTracing=False)

        context2.resumePausedComputation(computation)
        context2.resume()

        self.assertEqual(context2.getFinishedResult().asResult.result.pyval,
                         100000)

        context.teardown()
        context2.teardown()
Пример #36
0
 def setUp(self):
     self.callbackScheduler = CallbackScheduler.singletonForTesting()
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.compiler = self.runtime.getTypedForaCompiler()
     self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
def main(argv):
	fora.init_local()
	runtime = Runtime.getMainRuntime()
	handler = runtime.getInterpreterTraceHandler()

	handler.replayTracesFromFile(argv[1])
Пример #38
0
def main(argv):
    fora.init_local()
    runtime = Runtime.getMainRuntime()
    handler = runtime.getInterpreterTraceHandler()

    handler.replayTracesFromFile(argv[1])
Пример #39
0
    def __init__(self, callbackScheduler, sharedStateViewFactory, computedValueGatewayFactory):
        self.lock = threading.Lock()
        self.cacheLoadEvents = {}

        self.resultsById_ = {}
        self.eventsById_ = {}

        logging.info("created a component host")

        self.graph = ComputedGraph.ComputedGraph()

        logging.info("created a ComputedGraph")

        Runtime.initialize()
        logging.info("Runtime initialized")

        ModuleImporter.initialize()
        logging.info("Module importer initialized")

        Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())

        self.incomingObjectCache = IncomingObjectCache()
        self.outgoingObjectCache = OutgoingObjectCache()

        self.VDM = VectorDataManager.constructVDM(callbackScheduler)
        self.VDM.setDropUnreferencedPagesWhenFull(True)
        logging.info("created a VDM")

        logging.info("got shared state view factory: %s", sharedStateViewFactory)

        def initValueGateway():
            with self.graph:
                self.computedValueGateway = computedValueGatewayFactory()
                self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway

        def initSynchronizer():
            self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()

            logging.info("created a SharedStateSynchronizer")

            self.synchronizer.attachView(sharedStateViewFactory.createView())

            logging.info("attached shared state view.")

        simultaneously(initSynchronizer, initValueGateway)

        self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()

        self.outstandingMessagesById = {}
        self.expectedMessageId = 0

        self.messageTypeHandlers = {}

        self.messageTypeHandlers["Read"] = self.handleReadMessage
        self.messageTypeHandlers["Assign"] = self.handleAssignMessage
        self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
        self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
        self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds

        self.pendingObjectQueue = []

        self.subscriptions = Subscriptions.Subscriptions(self.graph, self.computedValueGateway, self.synchronizer)
Пример #40
0
    def __init__(self,
                 callbackScheduler,
                 sharedStateViewFactory,
                 computedValueGatewayFactory):
        self.lock = threading.Lock()
        self.cacheLoadEvents = {}

        self.resultsById_ = {}
        self.eventsById_ = {}

        logging.info("created a component host")

        self.graph = ComputedGraph.ComputedGraph()

        logging.info("created a ComputedGraph")

        Runtime.initialize()
        logging.info("Runtime initialized")

        ModuleImporter.initialize()
        logging.info("Module importer initialized")


        Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())

        self.incomingObjectCache = IncomingObjectCache()
        self.outgoingObjectCache = OutgoingObjectCache()

        self.VDM = VectorDataManager.constructVDM(callbackScheduler)
        self.VDM.setDropUnreferencedPagesWhenFull(True)
        logging.info("created a VDM")

        logging.info("got shared state view factory: %s", sharedStateViewFactory)

        def initValueGateway():
            with self.graph:
                self.computedValueGateway = computedValueGatewayFactory()
                self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway


        def initSynchronizer():
            self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()

            logging.info("created a SharedStateSynchronizer")

            self.synchronizer.attachView(
                sharedStateViewFactory.createView()
                )

            logging.info("attached shared state view.")

        simultaneously(
            initSynchronizer,
            initValueGateway
            )

        self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()

        self.outstandingMessagesById = {}
        self.expectedMessageId = 0

        self.messageTypeHandlers = {}

        self.messageTypeHandlers["Read"] = self.handleReadMessage
        self.messageTypeHandlers["Assign"] = self.handleAssignMessage
        self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
        self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
        self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds

        self.pendingObjectQueue = []

        self.subscriptions = Subscriptions.Subscriptions(
            self.graph,
            self.computedValueGateway,
            self.synchronizer
            )
Пример #41
0
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""
Run this script to generate a file of axiom signatures which can be fed into Axioms_consistency_test.py.
This generates all axioms, but by default comments them all out. Uncomment the ones you want to test.
"""

import ufora.native
import ufora.FORA.python.Runtime as Runtime
import ufora.native.FORA as FORANative
import os

dir = os.path.dirname(__file__)
AXIOMS_TO_TEST_FILENAME = os.path.join(dir, "AXIOMS_TO_TEST.txt")

runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()

readme_string = '"""\nThis file lists all the axioms we would like to check for consistency in\nAxioms_consistency_test.py. It supports basic python-like commenting\n"""\n\n'

with open(AXIOMS_TO_TEST_FILENAME, "w") as f:
    f.write(readme_string)
    f.write('"""\n')
    for i in range(axioms.axiomCount):
        f.write("%s\n" %axioms.getAxiomGroupByIndex(i).signature())
    f.write('"""')

Пример #42
0
    def __init__(self, ownAddress, channelListener, channelFactory,
                 eventHandler, callbackScheduler, diagnosticsDir, config,
                 viewFactory):
        Stoppable.Stoppable.__init__(self)

        #acquire a machineId randomly, using uuid
        self.machineId = CumulusNative.MachineId(
            Hash.Hash.sha1(str(uuid.uuid4())))

        self.ownAddress = ownAddress
        self.callbackScheduler = callbackScheduler
        self.viewFactory = viewFactory
        self.threadsStarted_ = False
        self.connectedMachines = set()
        self.connectingMachines = set(
        )  # machines we are in the process of connecting to
        self.droppedMachineIds = set()
        self.lock = threading.RLock()
        self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024 * 1024
        self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024 * 1024
        self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
        self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc

        self.reconnectPersistentCacheIndexViewThreads = []

        if config.cumulusDiskCacheStorageSubdirectory is not None:
            self.cumulusDiskCacheWantsDeletionOnTeardown = True
            self.cumulusDiskCacheStorageDir = os.path.join(
                config.cumulusDiskCacheStorageDir,
                config.cumulusDiskCacheStorageSubdirectory)
        else:
            self.cumulusDiskCacheWantsDeletionOnTeardown = False
            self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir

        logging.info(
            "Creating a CumulusService with ram cache of %s / %s MB and %s threads",
            self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusThreadCountOverride)

        self._stopEvent = threading.Event()

        self._channelListener = channelListener
        assert len(self._channelListener.ports) == 2
        self._channelFactory = channelFactory

        Runtime.initialize()
        ModuleImporter.initialize()

        self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
            self.viewFactory)

        self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
            target=self._channelListener.start)

        self.vdm = VectorDataManager.constructVDM(
            callbackScheduler, self.cumulusVectorRamCacheSizeOverride,
            self.cumulusMaxRamCacheSizeOverride)

        if self.cumulusTrackTcMalloc:
            logging.info(
                "CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
                self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0)
            self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()

        self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
            viewFactory.createView(retrySeconds=10.0, numRetries=10),
            callbackScheduler)

        self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)

        self.deleteCumulusDiskCacheIfNecessary()

        self.offlineCache = CumulusNative.DiskOfflineCache(
            callbackScheduler, self.cumulusDiskCacheStorageDir,
            config.cumulusDiskCacheStorageMB * 1024 * 1024,
            config.cumulusDiskCacheStorageFileCount)

        checkpointInterval = config.cumulusCheckpointIntervalSeconds
        if checkpointInterval == 0:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None ()
        else:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
                checkpointInterval, 1024 * 1024)

        self.cumulusWorker = self.constructCumlusWorker(
            callbackScheduler,
            CumulusNative.CumulusWorkerConfiguration(
                self.machineId,
                self.cumulusThreadCountOverride, checkpointPolicy,
                ExecutionContext.createContextConfiguration(), diagnosticsDir
                or ""), self.vdm, self.offlineCache, eventHandler)

        #externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
        #callbackScheduler
        #)
        #self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
        #settings.s3InterfaceFactory,
        #settings.objectStore,
        #self.vdm,
        #externalDatasetChannel.makeQueuelike(callbackScheduler)
        #)

        self.cumulusWorker.startComputations()
Пример #43
0
 def setUp(self):
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.native_runtime = self.runtime.getTypedForaCompiler()
     self.vdm = FORANative.VectorDataManager(
         CallbackScheduler.singletonForTesting(), 10000000)
Пример #44
0
 def setUp(self):
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.compiler = self.runtime.getTypedForaCompiler()
Пример #45
0
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""
rebuildAxiomSearchFunction

Generate appropriate axiom build functions after we've built the project successfully.
"""

import os
import ufora
import ufora.config.Setup as Setup
import ufora.FORA.python.Runtime as Runtime

Runtime.initialize(Setup.defaultSetup())

axioms = Runtime.getMainRuntime().getAxioms()

implvalCode, jovtCode = axioms.getCppWrapperCode()

uforaDir = os.path.split(os.path.abspath(ufora.__file__))[0]

with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch.cpp"), "w") as f:
    print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
    print >> f, implvalCode

with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch2.cpp"), "w") as f:
    print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
    print >> f, jovtCode
Пример #46
0
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""
rebuildAxiomSearchFunction

Generate appropriate axiom build functions after we've built the project successfully.
"""

import os
import ufora
import ufora.config.Setup as Setup
import ufora.FORA.python.Runtime as Runtime

Runtime.initialize(Setup.defaultSetup())

axioms = Runtime.getMainRuntime().getAxioms()

implvalCode, jovtCode = axioms.getCppWrapperCode()

uforaDir = os.path.split(os.path.abspath(ufora.__file__))[0]

with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch.cpp"),
          "w") as f:
    print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
    print >> f, implvalCode

with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch2.cpp"),
          "w") as f:
    print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
Пример #47
0
 def setUp(self):
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.native_runtime = self.runtime.getTypedForaCompiler()
     self.vdm = FORANative.VectorDataManager(CallbackScheduler.singletonForTesting(), 10000000)
Пример #48
0
 def setUp(self):
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.compiler = self.runtime.getTypedForaCompiler()
Пример #49
0
    def __init__(self,
                 ownAddress,
                 channelListener,
                 channelFactory,
                 eventHandler,
                 callbackScheduler,
                 diagnosticsDir,
                 config,
                 viewFactory,
                 s3InterfaceFactory=None,
                 objectStore=None):
        Stoppable.Stoppable.__init__(self)

        #acquire a machineId randomly, using uuid
        self.machineId = CumulusNative.MachineId(
            Hash.Hash.sha1(str(uuid.uuid4()))
            )

        self.ownAddress = ownAddress
        self.callbackScheduler = callbackScheduler
        self.viewFactory = viewFactory
        self.s3InterfaceFactory = s3InterfaceFactory
        self.objectStore = objectStore
        self.threadsStarted_ = False
        self.connectedMachines = set()
        self.connectingMachines = set()  # machines we are in the process of connecting to
        self.droppedMachineIds = set()
        self.lock = threading.RLock()
        self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
        self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
        self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
        self.cumulusTrackTcmalloc = config.cumulusTrackTcmalloc
        self.eventHandler = eventHandler

        self.reconnectPersistentCacheIndexViewThreads = []

        if config.cumulusDiskCacheStorageSubdirectory is not None:
            self.cumulusDiskCacheWantsDeletionOnTeardown = True
            self.cumulusDiskCacheStorageDir = os.path.join(
                config.cumulusDiskCacheStorageDir,
                config.cumulusDiskCacheStorageSubdirectory
                )
        else:
            self.cumulusDiskCacheWantsDeletionOnTeardown = False
            self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir

        self._stopEvent = threading.Event()

        self._channelListener = channelListener
        assert len(self._channelListener.ports) == 2
        self._channelFactory = channelFactory

        Runtime.initialize()
        ModuleImporter.initialize()

        self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
            self.viewFactory
            )

        self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
            target=self._channelListener.start
            )

        self.vdm = VectorDataManager.constructVDM(
            callbackScheduler,
            self.cumulusVectorRamCacheSizeOverride,
            self.cumulusMaxRamCacheSizeOverride
            )

        if self.cumulusTrackTcmalloc:
            self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()

        self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
            viewFactory.createView(retrySeconds=10.0, numRetries=10),
            callbackScheduler
            )

        self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)

        self.deleteCumulusDiskCacheIfNecessary()

        self.offlineCache = CumulusNative.DiskOfflineCache(
            callbackScheduler,
            self.cumulusDiskCacheStorageDir,
            config.cumulusDiskCacheStorageMB * 1024 * 1024,
            config.cumulusDiskCacheStorageFileCount
            )

        #If the "s3InterfaceFactory" is not in-memory, we use real out of process python.
        #it would be better if this were more explicit
        outOfProcess = self.s3InterfaceFactory is not None and self.s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool

        self.outOfProcessPythonTasks = OutOfProcessPythonTasks.OutOfProcessPythonTasks(outOfProcess=outOfProcess)

        self.vdm.initializeOutOfProcessPythonTasks(self.outOfProcessPythonTasks.nativeTasks)

        checkpointInterval = config.cumulusCheckpointIntervalSeconds
        if checkpointInterval == 0:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
        else:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
                checkpointInterval,
                1024 * 1024
                )

        self.cumulusWorker = self.constructCumlusWorker(
            callbackScheduler,
            CumulusNative.CumulusWorkerConfiguration(
                self.machineId,
                self.cumulusThreadCountOverride,
                checkpointPolicy,
                ExecutionContext.createContextConfiguration(),
                diagnosticsDir or ""
                ),
            self.vdm,
            self.offlineCache,
            eventHandler
            )

        self.datasetLoadService = None
        if self.s3InterfaceFactory:
            externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
                callbackScheduler
                )
            self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
                self.s3InterfaceFactory,
                self.objectStore,
                self.vdm,
                externalDatasetChannel.makeQueuelike(callbackScheduler)
                )

        self.cumulusWorker.startComputations()

        if self.datasetLoadService:
            self.datasetLoadService.startService()
Пример #50
0
    def __init__(self,
                 ownAddress,
                 channelListener,
                 channelFactory,
                 eventHandler,
                 callbackScheduler,
                 diagnosticsDir,
                 config,
                 viewFactory):
        Stoppable.Stoppable.__init__(self)

        #acquire a machineId randomly, using uuid
        self.machineId = CumulusNative.MachineId(
            Hash.Hash.sha1(str(uuid.uuid4()))
            )

        self.ownAddress = ownAddress
        self.callbackScheduler = callbackScheduler
        self.viewFactory = viewFactory
        self.threadsStarted_ = False
        self.connectedMachines = set()
        self.connectingMachines = set()  # machines we are in the process of connecting to
        self.droppedMachineIds = set()
        self.lock = threading.RLock()
        self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
        self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
        self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
        self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc

        self.reconnectPersistentCacheIndexViewThreads = []

        if config.cumulusDiskCacheStorageSubdirectory is not None:
            self.cumulusDiskCacheWantsDeletionOnTeardown = True
            self.cumulusDiskCacheStorageDir = os.path.join(
                config.cumulusDiskCacheStorageDir,
                config.cumulusDiskCacheStorageSubdirectory
                )
        else:
            self.cumulusDiskCacheWantsDeletionOnTeardown = False
            self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir


        logging.info(
            "Creating a CumulusService with ram cache of %s / %s MB and %s threads",
            self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusThreadCountOverride
            )

        self._stopEvent = threading.Event()

        self._channelListener = channelListener
        assert len(self._channelListener.ports) == 2
        self._channelFactory = channelFactory

        Runtime.initialize()
        ModuleImporter.initialize()

        self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
            self.viewFactory
            )

        self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
            target=self._channelListener.start
            )

        self.vdm = VectorDataManager.constructVDM(
            callbackScheduler,
            self.cumulusVectorRamCacheSizeOverride,
            self.cumulusMaxRamCacheSizeOverride
            )

        if self.cumulusTrackTcMalloc:
            logging.info(
                "CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
                self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0
                )
            self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()

        self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
            viewFactory.createView(retrySeconds=10.0, numRetries=10),
            callbackScheduler
            )

        self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)

        self.deleteCumulusDiskCacheIfNecessary()

        self.offlineCache = CumulusNative.DiskOfflineCache(
            callbackScheduler,
            self.cumulusDiskCacheStorageDir,
            config.cumulusDiskCacheStorageMB * 1024 * 1024,
            config.cumulusDiskCacheStorageFileCount
            )

        checkpointInterval = config.cumulusCheckpointIntervalSeconds
        if checkpointInterval == 0:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
        else:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
                checkpointInterval,
                1024 * 1024
                )

        self.cumulusWorker = self.constructCumlusWorker(
            callbackScheduler,
            CumulusNative.CumulusWorkerConfiguration(
                self.machineId,
                self.cumulusThreadCountOverride,
                checkpointPolicy,
                ExecutionContext.createContextConfiguration(),
                diagnosticsDir or ""
                ),
            self.vdm,
            self.offlineCache,
            eventHandler
            )

        #externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
            #callbackScheduler
            #)
        #self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
            #settings.s3InterfaceFactory,
            #settings.objectStore,
            #self.vdm,
            #externalDatasetChannel.makeQueuelike(callbackScheduler)
            #)

        self.cumulusWorker.startComputations()