Ejemplo n.º 1
0
def initialize(setupObjectToUse=None, useLocalEvaluator=True):
    global _builtin
    if _builtin is not None:
        return

    Runtime.initialize(setupObjectToUse)
    ModuleImporter.initialize(setupObjectToUse)
    Evaluator.initialize(setupObjectToUse, useLocalEvaluator)

    _builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
Ejemplo n.º 2
0
def main(parsedArguments):
    Runtime.initialize()

    testMissingMapFile()
    testMissingDatFiles()
    testMissingIndexFiles()

    testCorruptMapFile()
    testCorruptDatFiles()
    testCorruptIndexFiles()
Ejemplo n.º 3
0
def main(parsedArguments):
    Runtime.initialize()

    testMissingMapFile()
    testMissingDatFiles()
    testMissingIndexFiles()

    testCorruptMapFile()
    testCorruptDatFiles()
    testCorruptIndexFiles()
Ejemplo n.º 4
0
def initialize(setupObjectToUse = None, useLocalEvaluator = True):
    global _builtin
    if _builtin is not None:
        return

    Runtime.initialize(setupObjectToUse)
    ModuleImporter.initialize(setupObjectToUse)
    Evaluator.initialize(setupObjectToUse, useLocalEvaluator)

    _builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
Ejemplo n.º 5
0
def main(argv):
    t0 = time.time()

    Runtime.initialize(Setup.defaultSetup())

    runtime = Runtime.getMainRuntime()
    handler = runtime.getInterpreterTraceHandler()

    handler.replayTracesFromFile(argv[1])

    print "took ", time.time() - t0
def main(argv):
	t0 = time.time()

	Runtime.initialize(Setup.defaultSetup())
	
	runtime = Runtime.getMainRuntime()
	handler = runtime.getInterpreterTraceHandler()

	handler.replayTracesFromFile(argv[1])

	print "took ", time.time() - t0
Ejemplo n.º 7
0
def main(parsedArguments):
    Runtime.initialize()

    tester = AxiomTester(
        parsedArguments.numRandVals,
        parsedArguments.numRelaxations,
        parsedArguments.maxForRelax,
        parsedArguments.maxForRand,
        parsedArguments.testAxiomsPath,
        parsedArguments.seed
        )

    tester.test_axiom_consistency()
Ejemplo n.º 8
0
    def test_extractPausedComputationDuringVectorLoad(self):
        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(
            dataManager = vdm,
            allowInterpreterTracing = False
            )

        context.evaluate(
            FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
            FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
            )

        pagedVec = context.getFinishedResult().asResult.result

        context.placeInEvaluationState(
            FORANative.ImplValContainer(
                (pagedVec,
                FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
                FORANative.ImplValContainer(0))
                )
            )

        vdm.unloadAllPossible()

        context.resume()

        self.assertTrue(context.isVectorLoad())

        computation = context.extractPausedComputation()

        self.assertEqual(len(computation.frames),1)
Ejemplo n.º 9
0
 def setUp(self):
     self.callbackScheduler = CallbackScheduler.singletonForTesting()
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.compiler = self.runtime.getTypedForaCompiler()
     self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(
         FORA.builtin().implVal_)
Ejemplo n.º 10
0
    def __init__(self, numRandVals, numRelaxations, maxForRelax, maxForRand, testAxiomsPath, seed):
        object.__init__(self)
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.callbackSchedulerFactory = self.callbackScheduler.getFactory()
        
        self.numRandVals = numRandVals
        self.numRelaxations = numRelaxations
        self.maxForRelax = maxForRelax
        self.maxForRand = maxForRand
        self.seed = seed

        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.typed_fora_compiler = self.runtime.getTypedForaCompiler()

        if testAxiomsPath is not None:
            pathToUse = testAxiomsPath
        else:
            pathToUse = UNIT_TEST_AXIOMS_PATH

        self.axiom_signatures_to_test = self.loadAxiomSignaturesFromFile(pathToUse)

        self.axiom_groups = []
        for i in range(self.axioms.axiomCount):
            self.axiom_groups.append(self.axioms.getAxiomGroupByIndex(i))

        self.symbol_strings = self.loadSymbolStrings()

        numpy.random.seed(seed)
Ejemplo n.º 11
0
    def test_extractPausedComputationDuringVectorLoad(self):
        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler,
                                           Setup.config().maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(
            dataManager=vdm, allowInterpreterTracing=False)

        context.evaluate(
            FORA.extractImplValContainer(FORA.eval("fun() { [1,2,3].paged }")),
            FORANative.ImplValContainer(FORANative.makeSymbol("Call")))

        pagedVec = context.getFinishedResult().asResult.result

        context.placeInEvaluationState(
            FORANative.ImplValContainer(
                (pagedVec,
                 FORANative.ImplValContainer(FORANative.makeSymbol("GetItem")),
                 FORANative.ImplValContainer(0))))

        vdm.unloadAllPossible()

        context.resume()

        self.assertTrue(context.isVectorLoad())

        computation = context.extractPausedComputation()

        self.assertEqual(len(computation.frames), 1)
Ejemplo n.º 12
0
    def test_resumePausedComputationWithResult(self):
        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler,
                                           Setup.config().maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(
            dataManager=vdm,
            allowInterpreterTracing=False,
            allowInternalSplitting=False)

        text = """
        let f = fun(v, ix) {
            if (ix > 0)
                {
                let (v2,res) = f(v,ix-1);
                return (v2, res + v2[0])
                }

            `TriggerInterruptForTesting()

            return (v, 0)
            };

        f([1], 10)
        """

        evaluate(
            context,
            FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")),
            FORANative.ImplValContainer(FORANative.makeSymbol("Call")))

        assert context.isInterrupted()

        pausedComp = context.extractPausedComputation()

        framesToUse = pausedComp.asThread.computation.frames[0:5]

        pausedComp2 = FORANative.PausedComputationTree(
            FORANative.PausedComputation(
                framesToUse,
                FORA.extractImplValContainer(
                    FORA.eval("([2], 0)", keepAsForaValue=True)), False))

        context.resumePausedComputation(pausedComp2)

        context.copyValuesOutOfVectorPages()
        context.pageLargeVectorHandles(0)

        context.resetInterruptState()
        context.compute()

        self.assertTrue(context.isFinished())

        result = context.getFinishedResult()

        self.assertTrue(result.asResult.result[1].pyval == 6)
    def testMatchStructureCantMatchTooSmallTuple(self):
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()

        jov = FORANative.parseStringToJOV("((1,2,3),`StructureMatch,((nothing, false), (nothing, false), (nothing, false), (nothing, false)))")

        axiom = axioms.getAxiomByJOVT(runtime.getTypedForaCompiler(), jov.asTuple.jov)

        self.assertTrue(list(axiom.asNative.resultSignature.resultPart().vals) == [FORANative.parseStringToJOV("nothing")])
Ejemplo n.º 14
0
    def test_resumePausedComputationWithResult(self):
        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)

        context = ExecutionContext.ExecutionContext(
            dataManager = vdm,
            allowInterpreterTracing = False
            )

        text = """
        let f = fun(v, ix) {
            if (ix > 0)
                {
                let (v2,res) = f(v,ix-1);
                return (v2, res + v2[0])
                }

            `TriggerInterruptForTesting()

            return (v, 0)
            };

        f([1], 10)
        """

        context.evaluate(
            FORA.extractImplValContainer(FORA.eval("fun() { " + text + " }")),
            FORANative.ImplValContainer(FORANative.makeSymbol("Call"))
            )

        assert context.isInterrupted()

        pausedComp = context.extractPausedComputation()

        framesToUse = pausedComp.frames[0:5]

        pausedComp2 = FORANative.PausedComputation(
            framesToUse,
            FORA.extractImplValContainer(FORA.eval("([2], 0)", keepAsForaValue=True)),
            False
            )

        context.resumePausedComputation(pausedComp2)

        context.copyValuesOutOfVectorPages()
        context.pageLargeVectorHandles(0)

        context.resetInterruptState()
        context.resume()

        self.assertTrue( context.isFinished() )

        result = context.getFinishedResult()

        self.assertTrue(result.asResult.result[1].pyval == 6)
Ejemplo n.º 15
0
    def testAxiomLookups(self):
        axioms = Runtime.getMainRuntime().getAxioms()

        jmtsToTest = [axiom.signature() for axiom in axioms]

        for tupJmt in jmtsToTest:
            self.assertEqual(
                axioms.axiomSearchLinear(tupJmt),
                axioms.axiomSearchTree(tupJmt),
                "Didn't produce same results for %s" % str(tupJmt))
Ejemplo n.º 16
0
    def testAxiomLookups(self):
        axioms = Runtime.getMainRuntime().getAxioms()

        jmtsToTest = [axiom.signature() for axiom in axioms]

        for tupJmt in jmtsToTest:
            self.assertEqual(
                axioms.axiomSearchLinear(tupJmt),
                axioms.axiomSearchTree(tupJmt),
                "Didn't produce same results for %s" % str(tupJmt)
                )
Ejemplo n.º 17
0
    def triggerCompilation(self, *args):
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler = runtime.getTypedForaCompiler()
        instructionGraph = runtime.getInstructionGraph()
        
        reasoner = ForaNative.SimpleForwardReasoner(compiler, instructionGraph, axioms)
        
        frame = reasoner.reasonAboutComputationDefinition(self.cumulusComputationDefinition)

        logging.critical("Result was %s", frame.exits())
Ejemplo n.º 18
0
    def runSimpleEvaluation(self, inputType, outputType):
        mainRuntime = Runtime.getMainRuntime()
        foraCompiler = mainRuntime.getTypedForaCompiler()

        while foraCompiler.anyCompilingOrPending():
            time.sleep(.01)

        aParticularStringValue = ForaNative.ImplValContainer(aBigString)

        callable = self.generateSimpleCallable(inputType, outputType)

        jumpTarget = foraCompiler.compile(callable)

        import gc
        gc.collect()

        for passIndex in range(PASS_COUNT):
            #type values are memoized, so we can't assume that the value has a refcount
            # of exactly one
            totalStringCount = ForaNative.totalStringCount()
            totalImplvalCount = ForaNative.totalImplvalCount()

            anExecutionContext = ExecutionContext.ExecutionContext(
                dataManager = VectorDataManager.constructVDM(self.callbackScheduler)
                )

            anExecutionContext.evaluateFunctionPointer(jumpTarget, aParticularStringValue)

            self.assertTrue(anExecutionContext.isFinished())

            res = anExecutionContext.getFinishedResult()

            self.assertTrue(not res.isException())
            self.assertEqual(res.asResult.result, aParticularStringValue)

            anExecutionContext.teardown()

            res = None

            #verify final refcounts
            self.assertEqual(
                aParticularStringValue.getStringObjectRefcount(),
                1,
                "refcounts weren't maintained in %s->%s. %s != 1" % (
                    inputType,
                    outputType,
                    aParticularStringValue.getStringObjectRefcount()
                    )
                )
            self.assertEqual(
                (totalStringCount, totalImplvalCount),
                (ForaNative.totalStringCount(), ForaNative.totalImplvalCount()),
                "refcounts weren't maintained in " + str(inputType) + "->" + str(outputType)
                )
Ejemplo n.º 19
0
    def setUp(self):
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.compiler = self.runtime.getTypedForaCompiler()
        self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)

        pyforaPath = os.path.join(os.path.split(pyfora.__file__)[0], "fora/purePython")
        self.purePythonAsJOV = FORANative.JudgmentOnValue.Constant(FORA.importModule(pyforaPath).implVal_)
        
        self.instructionGraph = self.runtime.getInstructionGraph()
        self.reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.instructionGraph, self.axioms)
Ejemplo n.º 20
0
    def setUp(self):
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.compiler = self.runtime.getTypedForaCompiler()
        self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)

        pyforaPath = os.path.join(os.path.split(pyfora.__file__)[0], "fora/purePython")
        self.purePythonAsJOV = FORANative.JudgmentOnValue.Constant(FORA.importModule(pyforaPath).implVal_)

        self.instructionGraph = self.runtime.getInstructionGraph()
        self.reasoner = FORANative.SimpleForwardReasoner(self.compiler, self.instructionGraph, self.axioms)
Ejemplo n.º 21
0
def reasonAboutForaCode(module, testName):
    if reasoner[0] is None:
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler = runtime.getTypedForaCompiler()
        reasoner[0] = ForaNative.SimpleForwardReasoner(compiler, axioms, False)

    moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)
    frame = reasoner[0].reason(makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))

    dumpReasonerSummary(reasoner[0], frame)

    return True
Ejemplo n.º 22
0
    def triggerCompilation(self, *args):
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler = runtime.getTypedForaCompiler()
        instructionGraph = runtime.getInstructionGraph()

        reasoner = ForaNative.SimpleForwardReasoner(compiler, instructionGraph,
                                                    axioms)

        frame = reasoner.reasonAboutComputationDefinition(
            self.cumulusComputationDefinition)

        logging.critical("Result was %s", frame.exits())
Ejemplo n.º 23
0
    def runSimpleEvaluation(self, inputType, outputType):
        mainRuntime = Runtime.getMainRuntime()
        foraCompiler = mainRuntime.getTypedForaCompiler()

        while foraCompiler.anyCompilingOrPending():
            time.sleep(.01)

        aParticularStringValue = ForaNative.ImplValContainer(aBigString)

        callable = self.generateSimpleCallable(inputType, outputType)

        jumpTarget = foraCompiler.compile(callable)

        import gc
        gc.collect()

        for passIndex in range(PASS_COUNT):
            #type values are memoized, so we can't assume that the value has a refcount
            # of exactly one
            totalStringCount = ForaNative.totalStringCount()
            totalImplvalCount = ForaNative.totalImplvalCount()

            anExecutionContext = ExecutionContext.ExecutionContext(
                dataManager=VectorDataManager.constructVDM(
                    self.callbackScheduler))

            anExecutionContext.evaluateFunctionPointer(jumpTarget,
                                                       aParticularStringValue)

            self.assertTrue(anExecutionContext.isFinished())

            res = anExecutionContext.getFinishedResult()

            self.assertTrue(not res.isException())
            self.assertEqual(res.asResult.result, aParticularStringValue)

            anExecutionContext.teardown()

            res = None

            #verify final refcounts
            self.assertEqual(
                aParticularStringValue.getStringObjectRefcount(), 1,
                "refcounts weren't maintained in %s->%s. %s != 1" %
                (inputType, outputType,
                 aParticularStringValue.getStringObjectRefcount()))
            self.assertEqual((totalStringCount, totalImplvalCount),
                             (ForaNative.totalStringCount(),
                              ForaNative.totalImplvalCount()),
                             "refcounts weren't maintained in " +
                             str(inputType) + "->" + str(outputType))
Ejemplo n.º 24
0
def reasonAboutForaCode(module, testName):
    if reasoner[0] is None:
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler = runtime.getTypedForaCompiler()
        reasoner[0] = ForaNative.SimpleForwardReasoner(compiler, axioms, False)

    moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)
    frame = reasoner[0].reason(
        makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))

    dumpReasonerSummary(reasoner[0], frame)

    return True
Ejemplo n.º 25
0
    def setUp(self):
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.native_runtime = self.runtime.getTypedForaCompiler()

        symbol_strings_set = Set()
        self.symbol_strings = []
        for i in range(self.axioms.axiomCount):
            sig = str(self.axioms.getAxiomGroupByIndex(i).signature())
            for j in [l.strip(' ') for l in sig.strip("()").split(',')]:
                if re.search('^`', j):
                    if j not in symbol_strings_set:
                        symbol_strings_set.add(j)
                        self.symbol_strings.append(j.strip('`'))
Ejemplo n.º 26
0
    def testMatchStructureCantMatchTooSmallTuple(self):
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()

        jov = FORANative.parseStringToJOV(
            "((1,2,3),`StructureMatch,((nothing, false), (nothing, false), (nothing, false), (nothing, false)))"
        )

        axiom = axioms.getAxiomByJOVT(runtime.getTypedForaCompiler(),
                                      jov.asTuple.jov)

        self.assertTrue(
            list(axiom.asNative.resultSignature.resultPart().vals) ==
            [FORANative.parseStringToJOV("nothing")])
Ejemplo n.º 27
0
def reasonAboutForaCode(module, moduleName, testName):
    if reasoner[0] is None:
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler[0] = runtime.getTypedForaCompiler()
        instructionGraph = runtime.getInstructionGraph()
        reasoner[0] = ForaNative.SimpleForwardReasoner(compiler[0], instructionGraph, axioms)

    moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)

    t0 = time.time()
    frame = reasoner[0].reasonAboutApply(makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))
    print "reasoning ", moduleName + "." + testName, " took ", time.time() - t0

    dumpReasonerSummary(moduleName, testName, reasoner[0], frame)

    return True
Ejemplo n.º 28
0
    def test_extractStacktrace(self):
        text = """fun() {
            let x = 0;
            let v = [1,2,3]
            while (x < 100000) {
                x = x + 1
                }
            (x,v)
            }"""

        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
        context = ExecutionContext.ExecutionContext(
            dataManager = vdm,
            allowInterpreterTracing = False
            )

        context.interruptAfterCycleCount(1000)

        context.evaluate(
            FORA.extractImplValContainer(FORA.eval(text)),
            FORANative.symbol_Call
            )

        trace = context.extractStacktrace(True)

        self.assertTrue(len(trace) == 1)

        stacktrace = trace[0][0]
        frameData = trace[0][1]

        codeLocation = FORANative.getCodeLocation(stacktrace.getIDs()[0])
        self.assertTrue(codeLocation is not None)


        self.assertTrue(frameData.wasCompiled is False)
        self.assertTrue(len(frameData.inScopeValues) == 2)
        self.assertTrue([str(x) for x in frameData.inScopeValues.keys] == ['x', 'v'])

        values = [frameData.inScopeValues[x] for x in frameData.inScopeValues.keys]
        self.assertTrue(values[0].isInt64())
        self.assertTrue(values[1].isVector())

        context.teardown()
Ejemplo n.º 29
0
    def test_extractStacktrace(self):
        text = """fun() {
            let x = 0;
            let v = [1,2,3]
            while (x < 100000) {
                x = x + 1
                }
            (x,v)
            }"""

        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler,
                                           Setup.config().maxPageSizeInBytes)
        context = ExecutionContext.ExecutionContext(
            dataManager=vdm, allowInterpreterTracing=False)

        context.interruptAfterCycleCount(1000)

        context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
                         FORANative.symbol_Call)

        trace = context.extractStacktrace(True)

        self.assertTrue(len(trace) == 1)

        stacktrace = trace[0][0]
        frameData = trace[0][1]

        codeLocation = FORANative.getCodeLocation(stacktrace.getIDs()[0])
        self.assertTrue(codeLocation is not None)

        self.assertTrue(frameData.wasCompiled is False)
        self.assertTrue(len(frameData.inScopeValues) == 2)
        self.assertTrue([str(x)
                         for x in frameData.inScopeValues.keys] == ['x', 'v'])

        values = [
            frameData.inScopeValues[x] for x in frameData.inScopeValues.keys
        ]
        self.assertTrue(values[0].isInt64())
        self.assertTrue(values[1].isVector())

        context.teardown()
Ejemplo n.º 30
0
    def setUp(self):
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.native_runtime = self.runtime.getTypedForaCompiler()
        self.vals_to_test = self.loadValuesFromFile(os.path.join(os.path.split(__file__)[0],
                                                        "AxiomJOA_test.txt"))

        self.evaluator = LocalEvaluator.LocalEvaluator(
                            lambda vdm: CumulusNative.SimpleOfflineCache(self.callbackScheduler, 1000000000),
                            10000000,
                            maxPageSizeInBytes = 100000
                            )
        self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)

        self.knownModulesAsConstantJOVs = dict()
        self.knownModulesAsConstantJOVs["builtin"] = \
                FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
Ejemplo n.º 31
0
    def setUp(self):
        self.callbackScheduler = CallbackScheduler.singletonForTesting()
        self.runtime = Runtime.getMainRuntime()
        self.axioms = self.runtime.getAxioms()
        self.native_runtime = self.runtime.getTypedForaCompiler()
        self.vals_to_test = self.loadValuesFromFile(
            os.path.join(os.path.split(__file__)[0], "AxiomJOA_test.txt"))

        self.evaluator = LocalEvaluator.LocalEvaluator(
            lambda vdm: CumulusNative.SimpleOfflineCache(
                self.callbackScheduler, 1000000000),
            10000000,
            maxPageSizeInBytes=100000)
        self.oldEvaluator = Evaluator.swapEvaluator(self.evaluator)

        self.knownModulesAsConstantJOVs = dict()
        self.knownModulesAsConstantJOVs["builtin"] = \
                FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
Ejemplo n.º 32
0
    def runSimpleEvaluation(self, callable, arguments, validator):
        mainRuntime = Runtime.getMainRuntime()
        foraCompiler = mainRuntime.getTypedForaCompiler()
        jumpTarget = foraCompiler.compile(callable)

        while foraCompiler.anyCompilingOrPending():
            time.sleep(.01)

        gc.collect()

        for passIndex in range(PASS_COUNT):
            totalStringCount = ForaNative.totalStringCount()
            totalImplvalCount = ForaNative.totalImplvalCount()

            anExecutionContext = ExecutionContext.ExecutionContext(
                dataManager = VectorDataManager.constructVDM(self.callbackScheduler)
                )

            anExecutionContext.evaluateFunctionPointer(jumpTarget, *arguments)

            self.assertTrue(anExecutionContext.isFinished())

            res = anExecutionContext.getFinishedResult()

            self.assertTrue(validator(res.asResult.result),
                "invalid result in " + str(callable) + " with " + str(arguments) +
                ". got " + str(res)
                )

            res = None
            anExecutionContext.teardown()

            curRefs = (ForaNative.totalStringCount(), ForaNative.totalImplvalCount())

            self.assertEqual(
                (totalStringCount, totalImplvalCount),
                curRefs,
                "refcounts weren't maintained in " + str(callable) + " with " + str(arguments) +
                ". %s != %s" % (
                    (totalStringCount, totalImplvalCount),
                    curRefs
                    )
                )
Ejemplo n.º 33
0
    def test_extractPausedComputation(self):
        text = """fun() {
            let x = 0;
            while (x < 100000)
                x = x + 1
            x
            }"""

        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler, Setup.config().maxPageSizeInBytes)
        context = ExecutionContext.ExecutionContext(
            dataManager = vdm,
            allowInterpreterTracing = False
            )

        context.interruptAfterCycleCount(1010)

        context.evaluate(
            FORA.extractImplValContainer(FORA.eval(text)),
            FORANative.symbol_Call
            )

        computation = context.extractPausedComputation()

        self.assertEqual(len(computation.frames), 2)

        self.assertEqual(computation.frames[1].values[0].pyval, 335)

        context2 = ExecutionContext.ExecutionContext(
            dataManager = vdm,
            allowInterpreterTracing = False
            )

        context2.resumePausedComputation(computation)
        context2.resume()

        self.assertEqual(context2.getFinishedResult().asResult.result.pyval, 100000)

        context.teardown()
        context2.teardown()
Ejemplo n.º 34
0
def reasonAboutForaCode(module, moduleName, testName):
    if reasoner[0] is None:
        runtime = Runtime.getMainRuntime()
        axioms = runtime.getAxioms()
        compiler[0] = runtime.getTypedForaCompiler()
        instructionGraph = runtime.getInstructionGraph()
        reasoner[0] = ForaNative.SimpleForwardReasoner(compiler[0],
                                                       instructionGraph,
                                                       axioms)

    moduleJOV = ForaNative.JudgmentOnValue.Constant(module.implVal_)

    t0 = time.time()
    frame = reasoner[0].reasonAboutApply(
        makeJovt(moduleJOV, symbolJov("Member"), symbolJov(testName)))
    print "reasoning ", moduleName + "." + testName, " took ", time.time() - t0

    dumpReasonerSummary(moduleName, testName, reasoner[0], frame)

    return True
Ejemplo n.º 35
0
    def test_extractPausedComputation(self):
        text = """fun() {
            let x = 0;
            while (x < 100000)
                x = x + 1
            x
            }"""

        self.runtime = Runtime.getMainRuntime()
        #self.dynamicOptimizer = self.runtime.dynamicOptimizer

        vdm = FORANative.VectorDataManager(callbackScheduler,
                                           Setup.config().maxPageSizeInBytes)
        context = ExecutionContext.ExecutionContext(
            dataManager=vdm, allowInterpreterTracing=False)

        context.interruptAfterCycleCount(1010)

        context.evaluate(FORA.extractImplValContainer(FORA.eval(text)),
                         FORANative.symbol_Call)

        computation = context.extractPausedComputation()

        self.assertEqual(len(computation.frames), 2)

        self.assertEqual(computation.frames[1].values[0].pyval, 335)

        context2 = ExecutionContext.ExecutionContext(
            dataManager=vdm, allowInterpreterTracing=False)

        context2.resumePausedComputation(computation)
        context2.resume()

        self.assertEqual(context2.getFinishedResult().asResult.result.pyval,
                         100000)

        context.teardown()
        context2.teardown()
Ejemplo n.º 36
0
 def setUp(self):
     self.callbackScheduler = CallbackScheduler.singletonForTesting()
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.compiler = self.runtime.getTypedForaCompiler()
     self.builtinsAsJOV = FORANative.JudgmentOnValue.Constant(FORA.builtin().implVal_)
def main(argv):
	fora.init_local()
	runtime = Runtime.getMainRuntime()
	handler = runtime.getInterpreterTraceHandler()

	handler.replayTracesFromFile(argv[1])
Ejemplo n.º 38
0
def main(argv):
    fora.init_local()
    runtime = Runtime.getMainRuntime()
    handler = runtime.getInterpreterTraceHandler()

    handler.replayTracesFromFile(argv[1])
Ejemplo n.º 39
0
    def __init__(self, callbackScheduler, sharedStateViewFactory, computedValueGatewayFactory):
        self.lock = threading.Lock()
        self.cacheLoadEvents = {}

        self.resultsById_ = {}
        self.eventsById_ = {}

        logging.info("created a component host")

        self.graph = ComputedGraph.ComputedGraph()

        logging.info("created a ComputedGraph")

        Runtime.initialize()
        logging.info("Runtime initialized")

        ModuleImporter.initialize()
        logging.info("Module importer initialized")

        Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())

        self.incomingObjectCache = IncomingObjectCache()
        self.outgoingObjectCache = OutgoingObjectCache()

        self.VDM = VectorDataManager.constructVDM(callbackScheduler)
        self.VDM.setDropUnreferencedPagesWhenFull(True)
        logging.info("created a VDM")

        logging.info("got shared state view factory: %s", sharedStateViewFactory)

        def initValueGateway():
            with self.graph:
                self.computedValueGateway = computedValueGatewayFactory()
                self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway

        def initSynchronizer():
            self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()

            logging.info("created a SharedStateSynchronizer")

            self.synchronizer.attachView(sharedStateViewFactory.createView())

            logging.info("attached shared state view.")

        simultaneously(initSynchronizer, initValueGateway)

        self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()

        self.outstandingMessagesById = {}
        self.expectedMessageId = 0

        self.messageTypeHandlers = {}

        self.messageTypeHandlers["Read"] = self.handleReadMessage
        self.messageTypeHandlers["Assign"] = self.handleAssignMessage
        self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
        self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
        self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds

        self.pendingObjectQueue = []

        self.subscriptions = Subscriptions.Subscriptions(self.graph, self.computedValueGateway, self.synchronizer)
Ejemplo n.º 40
0
    def __init__(self,
                 callbackScheduler,
                 sharedStateViewFactory,
                 computedValueGatewayFactory):
        self.lock = threading.Lock()
        self.cacheLoadEvents = {}

        self.resultsById_ = {}
        self.eventsById_ = {}

        logging.info("created a component host")

        self.graph = ComputedGraph.ComputedGraph()

        logging.info("created a ComputedGraph")

        Runtime.initialize()
        logging.info("Runtime initialized")

        ModuleImporter.initialize()
        logging.info("Module importer initialized")


        Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())

        self.incomingObjectCache = IncomingObjectCache()
        self.outgoingObjectCache = OutgoingObjectCache()

        self.VDM = VectorDataManager.constructVDM(callbackScheduler)
        self.VDM.setDropUnreferencedPagesWhenFull(True)
        logging.info("created a VDM")

        logging.info("got shared state view factory: %s", sharedStateViewFactory)

        def initValueGateway():
            with self.graph:
                self.computedValueGateway = computedValueGatewayFactory()
                self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway


        def initSynchronizer():
            self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()

            logging.info("created a SharedStateSynchronizer")

            self.synchronizer.attachView(
                sharedStateViewFactory.createView()
                )

            logging.info("attached shared state view.")

        simultaneously(
            initSynchronizer,
            initValueGateway
            )

        self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()

        self.outstandingMessagesById = {}
        self.expectedMessageId = 0

        self.messageTypeHandlers = {}

        self.messageTypeHandlers["Read"] = self.handleReadMessage
        self.messageTypeHandlers["Assign"] = self.handleAssignMessage
        self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
        self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
        self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds

        self.pendingObjectQueue = []

        self.subscriptions = Subscriptions.Subscriptions(
            self.graph,
            self.computedValueGateway,
            self.synchronizer
            )
Ejemplo n.º 41
0
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""
Run this script to generate a file of axiom signatures which can be fed into Axioms_consistency_test.py.
This generates all axioms, but by default comments them all out. Uncomment the ones you want to test.
"""

import ufora.native
import ufora.FORA.python.Runtime as Runtime
import ufora.native.FORA as FORANative
import os

dir = os.path.dirname(__file__)
AXIOMS_TO_TEST_FILENAME = os.path.join(dir, "AXIOMS_TO_TEST.txt")

runtime = Runtime.getMainRuntime()
axioms = runtime.getAxioms()

readme_string = '"""\nThis file lists all the axioms we would like to check for consistency in\nAxioms_consistency_test.py. It supports basic python-like commenting\n"""\n\n'

with open(AXIOMS_TO_TEST_FILENAME, "w") as f:
    f.write(readme_string)
    f.write('"""\n')
    for i in range(axioms.axiomCount):
        f.write("%s\n" %axioms.getAxiomGroupByIndex(i).signature())
    f.write('"""')

Ejemplo n.º 42
0
    def __init__(self, ownAddress, channelListener, channelFactory,
                 eventHandler, callbackScheduler, diagnosticsDir, config,
                 viewFactory):
        Stoppable.Stoppable.__init__(self)

        #acquire a machineId randomly, using uuid
        self.machineId = CumulusNative.MachineId(
            Hash.Hash.sha1(str(uuid.uuid4())))

        self.ownAddress = ownAddress
        self.callbackScheduler = callbackScheduler
        self.viewFactory = viewFactory
        self.threadsStarted_ = False
        self.connectedMachines = set()
        self.connectingMachines = set(
        )  # machines we are in the process of connecting to
        self.droppedMachineIds = set()
        self.lock = threading.RLock()
        self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024 * 1024
        self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024 * 1024
        self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
        self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc

        self.reconnectPersistentCacheIndexViewThreads = []

        if config.cumulusDiskCacheStorageSubdirectory is not None:
            self.cumulusDiskCacheWantsDeletionOnTeardown = True
            self.cumulusDiskCacheStorageDir = os.path.join(
                config.cumulusDiskCacheStorageDir,
                config.cumulusDiskCacheStorageSubdirectory)
        else:
            self.cumulusDiskCacheWantsDeletionOnTeardown = False
            self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir

        logging.info(
            "Creating a CumulusService with ram cache of %s / %s MB and %s threads",
            self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusThreadCountOverride)

        self._stopEvent = threading.Event()

        self._channelListener = channelListener
        assert len(self._channelListener.ports) == 2
        self._channelFactory = channelFactory

        Runtime.initialize()
        ModuleImporter.initialize()

        self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
            self.viewFactory)

        self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
            target=self._channelListener.start)

        self.vdm = VectorDataManager.constructVDM(
            callbackScheduler, self.cumulusVectorRamCacheSizeOverride,
            self.cumulusMaxRamCacheSizeOverride)

        if self.cumulusTrackTcMalloc:
            logging.info(
                "CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
                self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0)
            self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()

        self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
            viewFactory.createView(retrySeconds=10.0, numRetries=10),
            callbackScheduler)

        self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)

        self.deleteCumulusDiskCacheIfNecessary()

        self.offlineCache = CumulusNative.DiskOfflineCache(
            callbackScheduler, self.cumulusDiskCacheStorageDir,
            config.cumulusDiskCacheStorageMB * 1024 * 1024,
            config.cumulusDiskCacheStorageFileCount)

        checkpointInterval = config.cumulusCheckpointIntervalSeconds
        if checkpointInterval == 0:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None ()
        else:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
                checkpointInterval, 1024 * 1024)

        self.cumulusWorker = self.constructCumlusWorker(
            callbackScheduler,
            CumulusNative.CumulusWorkerConfiguration(
                self.machineId,
                self.cumulusThreadCountOverride, checkpointPolicy,
                ExecutionContext.createContextConfiguration(), diagnosticsDir
                or ""), self.vdm, self.offlineCache, eventHandler)

        #externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
        #callbackScheduler
        #)
        #self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
        #settings.s3InterfaceFactory,
        #settings.objectStore,
        #self.vdm,
        #externalDatasetChannel.makeQueuelike(callbackScheduler)
        #)

        self.cumulusWorker.startComputations()
Ejemplo n.º 43
0
 def setUp(self):
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.native_runtime = self.runtime.getTypedForaCompiler()
     self.vdm = FORANative.VectorDataManager(
         CallbackScheduler.singletonForTesting(), 10000000)
Ejemplo n.º 44
0
 def setUp(self):
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.compiler = self.runtime.getTypedForaCompiler()
Ejemplo n.º 45
0
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

"""
rebuildAxiomSearchFunction

Generate appropriate axiom build functions after we've built the project successfully.
"""

import os
import ufora
import ufora.config.Setup as Setup
import ufora.FORA.python.Runtime as Runtime

Runtime.initialize(Setup.defaultSetup())

axioms = Runtime.getMainRuntime().getAxioms()

implvalCode, jovtCode = axioms.getCppWrapperCode()

uforaDir = os.path.split(os.path.abspath(ufora.__file__))[0]

with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch.cpp"), "w") as f:
    print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
    print >> f, implvalCode

with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch2.cpp"), "w") as f:
    print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
    print >> f, jovtCode
Ejemplo n.º 46
0
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.
"""
rebuildAxiomSearchFunction

Generate appropriate axiom build functions after we've built the project successfully.
"""

import os
import ufora
import ufora.config.Setup as Setup
import ufora.FORA.python.Runtime as Runtime

Runtime.initialize(Setup.defaultSetup())

axioms = Runtime.getMainRuntime().getAxioms()

implvalCode, jovtCode = axioms.getCppWrapperCode()

uforaDir = os.path.split(os.path.abspath(ufora.__file__))[0]

with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch.cpp"),
          "w") as f:
    print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
    print >> f, implvalCode

with open(os.path.join(uforaDir, "FORA", "Axioms", "AxiomSearch2.cpp"),
          "w") as f:
    print >> f, "//generated by ufora/scripts/rebuildAxiomSearchFunction.py"
Ejemplo n.º 47
0
 def setUp(self):
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.native_runtime = self.runtime.getTypedForaCompiler()
     self.vdm = FORANative.VectorDataManager(CallbackScheduler.singletonForTesting(), 10000000)
Ejemplo n.º 48
0
 def setUp(self):
     self.runtime = Runtime.getMainRuntime()
     self.axioms = self.runtime.getAxioms()
     self.compiler = self.runtime.getTypedForaCompiler()
Ejemplo n.º 49
0
    def __init__(self,
                 ownAddress,
                 channelListener,
                 channelFactory,
                 eventHandler,
                 callbackScheduler,
                 diagnosticsDir,
                 config,
                 viewFactory,
                 s3InterfaceFactory=None,
                 objectStore=None):
        Stoppable.Stoppable.__init__(self)

        #acquire a machineId randomly, using uuid
        self.machineId = CumulusNative.MachineId(
            Hash.Hash.sha1(str(uuid.uuid4()))
            )

        self.ownAddress = ownAddress
        self.callbackScheduler = callbackScheduler
        self.viewFactory = viewFactory
        self.s3InterfaceFactory = s3InterfaceFactory
        self.objectStore = objectStore
        self.threadsStarted_ = False
        self.connectedMachines = set()
        self.connectingMachines = set()  # machines we are in the process of connecting to
        self.droppedMachineIds = set()
        self.lock = threading.RLock()
        self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
        self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
        self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
        self.cumulusTrackTcmalloc = config.cumulusTrackTcmalloc
        self.eventHandler = eventHandler

        self.reconnectPersistentCacheIndexViewThreads = []

        if config.cumulusDiskCacheStorageSubdirectory is not None:
            self.cumulusDiskCacheWantsDeletionOnTeardown = True
            self.cumulusDiskCacheStorageDir = os.path.join(
                config.cumulusDiskCacheStorageDir,
                config.cumulusDiskCacheStorageSubdirectory
                )
        else:
            self.cumulusDiskCacheWantsDeletionOnTeardown = False
            self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir

        self._stopEvent = threading.Event()

        self._channelListener = channelListener
        assert len(self._channelListener.ports) == 2
        self._channelFactory = channelFactory

        Runtime.initialize()
        ModuleImporter.initialize()

        self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
            self.viewFactory
            )

        self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
            target=self._channelListener.start
            )

        self.vdm = VectorDataManager.constructVDM(
            callbackScheduler,
            self.cumulusVectorRamCacheSizeOverride,
            self.cumulusMaxRamCacheSizeOverride
            )

        if self.cumulusTrackTcmalloc:
            self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()

        self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
            viewFactory.createView(retrySeconds=10.0, numRetries=10),
            callbackScheduler
            )

        self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)

        self.deleteCumulusDiskCacheIfNecessary()

        self.offlineCache = CumulusNative.DiskOfflineCache(
            callbackScheduler,
            self.cumulusDiskCacheStorageDir,
            config.cumulusDiskCacheStorageMB * 1024 * 1024,
            config.cumulusDiskCacheStorageFileCount
            )

        #If the "s3InterfaceFactory" is not in-memory, we use real out of process python.
        #it would be better if this were more explicit
        outOfProcess = self.s3InterfaceFactory is not None and self.s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool

        self.outOfProcessPythonTasks = OutOfProcessPythonTasks.OutOfProcessPythonTasks(outOfProcess=outOfProcess)

        self.vdm.initializeOutOfProcessPythonTasks(self.outOfProcessPythonTasks.nativeTasks)

        checkpointInterval = config.cumulusCheckpointIntervalSeconds
        if checkpointInterval == 0:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
        else:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
                checkpointInterval,
                1024 * 1024
                )

        self.cumulusWorker = self.constructCumlusWorker(
            callbackScheduler,
            CumulusNative.CumulusWorkerConfiguration(
                self.machineId,
                self.cumulusThreadCountOverride,
                checkpointPolicy,
                ExecutionContext.createContextConfiguration(),
                diagnosticsDir or ""
                ),
            self.vdm,
            self.offlineCache,
            eventHandler
            )

        self.datasetLoadService = None
        if self.s3InterfaceFactory:
            externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
                callbackScheduler
                )
            self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
                self.s3InterfaceFactory,
                self.objectStore,
                self.vdm,
                externalDatasetChannel.makeQueuelike(callbackScheduler)
                )

        self.cumulusWorker.startComputations()

        if self.datasetLoadService:
            self.datasetLoadService.startService()
Ejemplo n.º 50
0
    def __init__(self,
                 ownAddress,
                 channelListener,
                 channelFactory,
                 eventHandler,
                 callbackScheduler,
                 diagnosticsDir,
                 config,
                 viewFactory):
        Stoppable.Stoppable.__init__(self)

        #acquire a machineId randomly, using uuid
        self.machineId = CumulusNative.MachineId(
            Hash.Hash.sha1(str(uuid.uuid4()))
            )

        self.ownAddress = ownAddress
        self.callbackScheduler = callbackScheduler
        self.viewFactory = viewFactory
        self.threadsStarted_ = False
        self.connectedMachines = set()
        self.connectingMachines = set()  # machines we are in the process of connecting to
        self.droppedMachineIds = set()
        self.lock = threading.RLock()
        self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
        self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
        self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
        self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc

        self.reconnectPersistentCacheIndexViewThreads = []

        if config.cumulusDiskCacheStorageSubdirectory is not None:
            self.cumulusDiskCacheWantsDeletionOnTeardown = True
            self.cumulusDiskCacheStorageDir = os.path.join(
                config.cumulusDiskCacheStorageDir,
                config.cumulusDiskCacheStorageSubdirectory
                )
        else:
            self.cumulusDiskCacheWantsDeletionOnTeardown = False
            self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir


        logging.info(
            "Creating a CumulusService with ram cache of %s / %s MB and %s threads",
            self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusThreadCountOverride
            )

        self._stopEvent = threading.Event()

        self._channelListener = channelListener
        assert len(self._channelListener.ports) == 2
        self._channelFactory = channelFactory

        Runtime.initialize()
        ModuleImporter.initialize()

        self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
            self.viewFactory
            )

        self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
            target=self._channelListener.start
            )

        self.vdm = VectorDataManager.constructVDM(
            callbackScheduler,
            self.cumulusVectorRamCacheSizeOverride,
            self.cumulusMaxRamCacheSizeOverride
            )

        if self.cumulusTrackTcMalloc:
            logging.info(
                "CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
                self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0
                )
            self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()

        self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
            viewFactory.createView(retrySeconds=10.0, numRetries=10),
            callbackScheduler
            )

        self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)

        self.deleteCumulusDiskCacheIfNecessary()

        self.offlineCache = CumulusNative.DiskOfflineCache(
            callbackScheduler,
            self.cumulusDiskCacheStorageDir,
            config.cumulusDiskCacheStorageMB * 1024 * 1024,
            config.cumulusDiskCacheStorageFileCount
            )

        checkpointInterval = config.cumulusCheckpointIntervalSeconds
        if checkpointInterval == 0:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
        else:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
                checkpointInterval,
                1024 * 1024
                )

        self.cumulusWorker = self.constructCumlusWorker(
            callbackScheduler,
            CumulusNative.CumulusWorkerConfiguration(
                self.machineId,
                self.cumulusThreadCountOverride,
                checkpointPolicy,
                ExecutionContext.createContextConfiguration(),
                diagnosticsDir or ""
                ),
            self.vdm,
            self.offlineCache,
            eventHandler
            )

        #externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
            #callbackScheduler
            #)
        #self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
            #settings.s3InterfaceFactory,
            #settings.objectStore,
            #self.vdm,
            #externalDatasetChannel.makeQueuelike(callbackScheduler)
            #)

        self.cumulusWorker.startComputations()