コード例 #1
0
ファイル: CumulusService.py プロジェクト: nkhuyu/ufora
        def performConnect():
            clientOrMachine, guid = clientOrMachineAndGuid

            logging.info("Connection is available to %s with guid %s", clientOrMachine, guid)

            with self.lock:
                if self.shouldStop():
                    logging.info("Rejecting a connection because we are no longer active")
                    for channel in channels:
                        channel.disconnect()
                    return

            if clientOrMachine.isMachine():
                with self.lock:
                    if clientOrMachine.asMachine.machine in self.droppedMachineIds:
                        return
                    self.connectedMachines.add(clientOrMachine.asMachine.machine)

                    self.cumulusWorker.addMachine(
                        clientOrMachine.asMachine.machine,
                        channels,
                        ModuleImporter.builtinModuleImplVal(),
                        self.callbackScheduler
                        )
            else:
                self.cumulusWorker.addCumulusClient(
                    clientOrMachine.asClient.client,
                    channels,
                    ModuleImporter.builtinModuleImplVal(),
                    self.callbackScheduler
                    )
コード例 #2
0
ファイル: CumulusService.py プロジェクト: ufora/ufora
        def performConnect():
            clientOrMachine, guid = clientOrMachineAndGuid

            logging.info("Connection is available to %s with guid %s", clientOrMachine, guid)

            with self.lock:
                if self.shouldStop():
                    logging.info("Rejecting a connection because we are no longer active")
                    for channel in channels:
                        channel.disconnect()
                    return

            if clientOrMachine.isMachine():
                with self.lock:
                    if clientOrMachine.asMachine.machine in self.droppedMachineIds:
                        return
                    self.connectedMachines.add(clientOrMachine.asMachine.machine)

                    self.cumulusWorker.addMachine(
                        clientOrMachine.asMachine.machine,
                        channels,
                        ModuleImporter.builtinModuleImplVal(),
                        self.callbackScheduler
                        )
            else:
                self.cumulusWorker.addCumulusClient(
                    clientOrMachine.asClient.client,
                    channels,
                    ModuleImporter.builtinModuleImplVal(),
                    self.callbackScheduler
                    )
コード例 #3
0
ファイル: CumulusGatewayRemote.py プロジェクト: ufora/ufora
    def __init__(self,
                 callbackScheduler,
                 vdm,
                 channelFactory,
                 activeMachines,
                 viewFactory):
        CumulusGateway.CumulusGateway.__init__(self,
                                               callbackScheduler,
                                               vdm,
                                               viewFactory)

        ModuleImporter.initialize()

        self.channelFactory_ = channelFactory

        self.connectedMachines_ = set()
        self.disconnectedMachines_ = set()
        self.desiredMachines_ = set()

        self.connectingThreads_ = []
        self.isTornDown_ = False

        self.activeMachines = activeMachines
        self.activeMachines.addListener(self)

        self.activeMachines.startService()
コード例 #4
0
ファイル: CumulusService.py プロジェクト: nkhuyu/ufora
    def connectToRemoteWorker(self, machineId, ip, port, guid):
        logging.info("Attempting to connect to machine %s on %s:%s with guid %s",
                     machineId,
                     ip,
                     port,
                     guid)
        stringChannel = self._channelFactory.createChannel((ip, port))

        stringChannel.write(ufora.version)

        stringChannel.write(machineId.__getstate__())

        #initiate the handshake
        stringChannel.write(
            CumulusNative.CumulusClientOrMachine.Machine(
                self.machineId
                ).__getstate__()
            )

        stringChannel.write(guid.__getstate__())

        logging.info("CumulusService %s wrote handshake for %s with guid %s",
            self.machineId,
            machineId,
            guid
            )

        channelAsQueue = stringChannel.makeQueuelike(self.callbackScheduler)

        msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)

        if msg is None:
            logging.error("While attempting to add worker %s with guid %s, " +
                "CumulusWorker %s did not receive a builtin hash message during handshake",
                machineId,
                guid,
                self.machineId
                )
            return None

        otherWorkersBuiltinHash = Hash.Hash(0)
        otherWorkersBuiltinHash.__setstate__(msg)

        builtinsAgree = otherWorkersBuiltinHash == ModuleImporter.builtinModuleImplVal().hash

        if not builtinsAgree:
            logging.critical("CumulusWorker %s could not connect to CumulusWorker %s as they have " + \
                             "different builtins; former's builtin hash: %s, latter's builtin hash: " + \
                             "%s",
                             self.machineId,
                             machineId,
                             ModuleImporter.builtinModuleImplVal().hash,
                             otherWorkersBuiltinHash
                             )

            channelAsQueue.disconnect()
            return None

        return channelAsQueue
コード例 #5
0
ファイル: FORA.py プロジェクト: Sandy4321/ufora
def initialize(setupObjectToUse = None, useLocalEvaluator = True):
    global _builtin
    if _builtin is not None:
        return

    Runtime.initialize(setupObjectToUse)
    ModuleImporter.initialize(setupObjectToUse)
    Evaluator.initialize(setupObjectToUse, useLocalEvaluator)

    _builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
コード例 #6
0
ファイル: FORA.py プロジェクト: vishnur/ufora
def importModule(modulePath):
    #TODO BUG anybody:  why is this here? It was getting passed as the
    #searchForFreeVariables argument to importModuleFromPath for some reason
    ModuleImporter.builtinModuleImplVal()
    return ForaValue.FORAValue(
                ModuleImporter.importModuleFromPath(
                    modulePath,
                    True
                    )
                )
コード例 #7
0
def initialize(setupObjectToUse=None, useLocalEvaluator=True):
    global _builtin
    if _builtin is not None:
        return

    Runtime.initialize(setupObjectToUse)
    ModuleImporter.initialize(setupObjectToUse)
    Evaluator.initialize(setupObjectToUse, useLocalEvaluator)

    _builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
コード例 #8
0
    def connectToRemoteWorker(self, machineId, ip, port, guid):
        logging.info(
            "Attempting to connect to machine %s on %s:%s with guid %s",
            machineId, ip, port, guid)
        stringChannel = self._channelFactory.createChannel((ip, port))

        stringChannel.write(ufora.version)

        stringChannel.write(machineId.__getstate__())

        #initiate the handshake
        stringChannel.write(
            CumulusNative.CumulusClientOrMachine.Machine(
                self.machineId).__getstate__())

        stringChannel.write(guid.__getstate__())

        logging.info("CumulusService %s wrote handshake for %s with guid %s",
                     self.machineId, machineId, guid)

        channelAsQueue = stringChannel.makeQueuelike(self.callbackScheduler)

        msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)

        if msg is None:
            logging.error(
                "While attempting to add worker %s with guid %s, " +
                "CumulusWorker %s did not receive a builtin hash message during handshake",
                machineId, guid, self.machineId)
            return None

        otherWorkersBuiltinHash = Hash.Hash(0)
        otherWorkersBuiltinHash.__setstate__(msg)

        builtinsAgree = otherWorkersBuiltinHash == ModuleImporter.builtinModuleImplVal(
        ).hash

        if not builtinsAgree:
            logging.critical("CumulusWorker %s could not connect to CumulusWorker %s as they have " + \
                             "different builtins; former's builtin hash: %s, latter's builtin hash: " + \
                             "%s",
                             self.machineId,
                             machineId,
                             ModuleImporter.builtinModuleImplVal().hash,
                             otherWorkersBuiltinHash
                             )

            channelAsQueue.disconnect()
            return None

        return channelAsQueue
コード例 #9
0
    def __init__(self, callbackScheduler, channelListener, sharedStateAddress):
        Stoppable.Stoppable.__init__(self)
        self._lock = threading.Lock()
        self.callbackScheduler = callbackScheduler
        self.sharedStateAddress = sharedStateAddress

        self.channelListener = channelListener
        self.channelListener.registerConnectCallback(self.onSubscribableConnection)

        ModuleImporter.initialize()

        self.socketsToDisconnectOnExit = []
        self.procsToKillOnExit = set()
        self.isTornDown_ = False

        self.cleanupThread = ManagedThread.ManagedThread(target=self.cleanupThread_)
コード例 #10
0
ファイル: CumulusGatewayRemote.py プロジェクト: vishnur/ufora
    def connectToWorker(self, machineId, ip, port, guid):
        with self.lock_:
            stringChannel = self.channelFactory_.createChannel((ip, port))
            builtinsHash = ModuleImporter.builtinModuleImplVal().hash
            clientId = self.cumulusClientId
            callbackScheduler = self.callbackScheduler

        logging.info("Client %s writing version message '%s' to %s", clientId, ufora.version, machineId)

        stringChannel.write(ufora.version)

        logging.info("Client %s writing client ID message to %s", clientId, machineId)

        stringChannel.write(machineId.__getstate__())

        logging.info("Client %s writing expected machineId message to %s", clientId, machineId)

        stringChannel.write(CumulusNative.CumulusClientOrMachine.Client(clientId).__getstate__())

        logging.info("Client %s writing guid %s to %s", clientId, guid, machineId)
        stringChannel.write(guid.__getstate__())

        channelAsQueue = stringChannel.makeQueuelike(callbackScheduler)
        msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)
        if msg is None:
            logging.error(
                "While attempting to add worker %s, CumulusClient %s did not "
                + "receive a builtin hash message during handshake",
                machineId,
                clientId,
            )

        assert msg is not None

        logging.info("Client %s received serialized worker's builtin hash", clientId)

        try:
            workersBuiltinHash = HashNative.Hash(0)
            workersBuiltinHash.__setstate__(msg)
        except:
            logging.info("Client received a bad worker hash: %s of size %s", repr(msg), len(msg))
            raise

        builtinsAgree = workersBuiltinHash == builtinsHash

        if not builtinsAgree:
            logging.critical(
                "Could not connect CumulusClient %s to CumulusWorker %s as they "
                + "have different builtins; client's builtin hash: %s, worker's "
                + "builtin hash: %s. Disconnecting channel",
                clientId,
                machineId,
                builtinsHash,
                workersBuiltinHash,
            )

            channelAsQueue.disconnect()
            return None

        return channelAsQueue
コード例 #11
0
    def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm, **kwds):
        s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory()

        simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
            1,
            0,
            s3Service=s3Service,
            memoryPerWorkerMB=400,
            callbackScheduler=callbackScheduler,
            threadsPerWorker=Setup.config().cumulusServiceThreadCount,
            **kwds
            )

        CumulusGateway.CumulusGateway.__init__(self,
                                               callbackScheduler,
                                               vdm,
                                               simulation.sharedStateViewFactory)

        self.s3Service = s3Service
        self.simulation = simulation

        self.viewFactory = self.simulation.sharedStateViewFactory
        self.worker = self.simulation.getWorker(0)
        self.workerVdm = self.simulation.getWorkerVdm(0)

        channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
        channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)

        machineId = self.workerVdm.getMachineId()

        self.cumulusClient.addMachine(
            machineId,
            [channel1Client, channel2Client],
            ModuleImporter.builtinModuleImplVal(),
            self.callbackScheduler
            )

        self.worker.addCumulusClient(
            self.cumulusClientId,
            [channel1Worker, channel2Worker],
            ModuleImporter.builtinModuleImplVal(),
            self.callbackScheduler
            )

        self.loadingService = self.simulation.loadingServices[0]
コード例 #12
0
def constructConverter(purePythonModuleImplVal, vdm):
    pythonNameToPyforaName = {}

    pythonNameToPyforaName.update(NamedSingletons.pythonNameToPyforaName)
    pythonNameToPyforaName.update(PyAbortSingletons.pythonNameToPyforaName)

    return ForaNative.PythonBinaryStreamToImplval(
        vdm, purePythonModuleImplVal, ModuleImporter.builtinModuleImplVal(),
        pythonNameToPyforaName, PythonAstConverter.parseStringToPythonAst)
コード例 #13
0
    def __init__(self, callbackScheduler, channelListener, sharedStateAddress):
        Stoppable.Stoppable.__init__(self)
        self._lock = threading.Lock()
        self.callbackScheduler = callbackScheduler
        self.sharedStateAddress = sharedStateAddress

        self.channelListener = channelListener
        self.channelListener.registerConnectCallback(
            self.onSubscribableConnection)

        ModuleImporter.initialize()

        self.socketsToDisconnectOnExit = []
        self.procsToKillOnExit = set()
        self.isTornDown_ = False

        self.cleanupThread = ManagedThread.ManagedThread(
            target=self.cleanupThread_)
コード例 #14
0
ファイル: CumulusService.py プロジェクト: ufora/ufora
    def onWorkerAdd2(self, machineId, ip, ports, guid):
        logging.debug(
            "CumulusService %s adding worker %s and connecting to it with guid %s",
            self.machineId,
            machineId,
            guid
            )

        assert len(ports) == 2
        channels = []
        while self.shouldConnectToMachine(machineId):
            try:
                for channel in channels:
                    channel.disconnect()
                channels = []

                for port in ports:
                    channel = None
                    tries = 0

                    while channel is None and tries < 3:
                        channel = self.connectToRemoteWorker(machineId, ip, port, guid)
                        tries += 1

                    if channel is None:
                        logging.error(
                            "Unable to connect to worker %s on %s:%s with guid %s.",
                            machineId,
                            ip,
                            port,
                            guid
                            )
                        continue

                    channels.append(channel)

                with self.lock:
                    if machineId in self.droppedMachineIds:
                        logging.warn("not accepting %s", machineId)
                    self.connectedMachines.add(machineId)
                    self.connectingMachines.discard(machineId)

                    self.cumulusWorker.addMachine(
                        machineId,
                        channels,
                        ModuleImporter.builtinModuleImplVal(),
                        self.callbackScheduler
                        )

                logging.info("Connection is available to Worker %s with guid %s",
                             machineId,
                             guid)
                return
            except:
                logging.error("Failed to add worker %s: %s", machineId, traceback.format_exc())
コード例 #15
0
ファイル: CumulusService.py プロジェクト: ufora/ufora
    def onWorkerAdd2(self, machineId, ip, ports, guid):
        logging.debug(
            "CumulusService %s adding worker %s and connecting to it with guid %s",
            self.machineId,
            machineId,
            guid
            )

        assert len(ports) == 2
        channels = []
        while self.shouldConnectToMachine(machineId):
            try:
                for channel in channels:
                    channel.disconnect()
                channels = []

                for port in ports:
                    channel = None
                    tries = 0

                    while channel is None and tries < 3:
                        channel = self.connectToRemoteWorker(machineId, ip, port, guid)
                        tries += 1

                    if channel is None:
                        logging.error(
                            "Unable to connect to worker %s on %s:%s with guid %s.",
                            machineId,
                            ip,
                            port,
                            guid
                            )
                        continue

                    channels.append(channel)

                with self.lock:
                    if machineId in self.droppedMachineIds:
                        logging.warn("not accepting %s", machineId)
                    self.connectedMachines.add(machineId)
                    self.connectingMachines.discard(machineId)

                    self.cumulusWorker.addMachine(
                        machineId,
                        channels,
                        ModuleImporter.builtinModuleImplVal(),
                        self.callbackScheduler
                        )

                logging.info("Connection is available to Worker %s with guid %s",
                             machineId,
                             guid)
                return
            except:
                logging.error("Failed to add worker %s: %s", machineId, traceback.format_exc())
コード例 #16
0
    def __init__(self, callbackScheduler, vdm, channelFactory, activeMachines,
                 viewFactory):
        CumulusGateway.CumulusGateway.__init__(self, callbackScheduler, vdm,
                                               viewFactory)

        ModuleImporter.initialize()

        self.channelFactory_ = channelFactory

        self.connectedMachines_ = set()
        self.disconnectedMachines_ = set()
        self.desiredMachines_ = set()

        self.connectingThreads_ = []
        self.isTornDown_ = False

        self.activeMachines = activeMachines
        self.activeMachines.addListener(self)

        self.activeMachines.startService()
コード例 #17
0
    def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm,
                 **kwds):
        s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory()

        simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
            1,
            0,
            s3Service=s3Service,
            memoryPerWorkerMB=400,
            callbackScheduler=callbackScheduler,
            threadsPerWorker=Setup.config().cumulusServiceThreadCount,
            **kwds)

        CumulusGateway.CumulusGateway.__init__(
            self, callbackScheduler, vdm, simulation.sharedStateViewFactory)

        self.s3Service = s3Service
        self.simulation = simulation

        self.viewFactory = self.simulation.sharedStateViewFactory
        self.worker = self.simulation.getWorker(0)
        self.workerVdm = self.simulation.getWorkerVdm(0)

        channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel(
            self.callbackScheduler)
        channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel(
            self.callbackScheduler)

        machineId = self.workerVdm.getMachineId()

        self.cumulusClient.addMachine(machineId,
                                      [channel1Client, channel2Client],
                                      ModuleImporter.builtinModuleImplVal(),
                                      self.callbackScheduler)

        self.worker.addCumulusClient(self.cumulusClientId,
                                     [channel1Worker, channel2Worker],
                                     ModuleImporter.builtinModuleImplVal(),
                                     self.callbackScheduler)

        self.loadingService = self.simulation.loadingServices[0]
コード例 #18
0
ファイル: Converter.py プロジェクト: ufora/ufora
def canonicalPurePythonModule():
    if canonicalPurePythonModuleCache_[0] is None:
        path = os.path.join(os.path.abspath(os.path.split(pyfora.__file__)[0]), "fora")
        moduleTree = ModuleDirectoryStructure.ModuleDirectoryStructure.read(path, "purePython", "fora")
        
        canonicalPurePythonModuleCache_[0] = ModuleImporter.importModuleFromMDS(
            moduleTree,
            "fora",
            "purePython",
            searchForFreeVariables=True
            )

    return canonicalPurePythonModuleCache_[0]
コード例 #19
0
ファイル: Converter.py プロジェクト: ufora/ufora
def constructConverter(purePythonModuleImplval, vdm):
    if purePythonModuleImplval is None:
        return Converter(vdmOverride=vdm)
    else:
        singletonAndExceptionConverter = \
            PyforaSingletonAndExceptionConverter.PyforaSingletonAndExceptionConverter(
                purePythonModuleImplval
                )

        primitiveTypeMapping = {
            bool: purePythonModuleImplval.getObjectMember("PyBool"),
            str: purePythonModuleImplval.getObjectMember("PyString"),
            int: purePythonModuleImplval.getObjectMember("PyInt"),
            float: purePythonModuleImplval.getObjectMember("PyFloat"),
            type(None): purePythonModuleImplval.getObjectMember("PyNone"),
            }


        nativeConstantConverter = ForaNative.PythonConstantConverter(
            primitiveTypeMapping
            )

        nativeListConverter = ForaNative.makePythonListConverter(
            purePythonModuleImplval.getObjectMember("PyList")
            )

        nativeTupleConverter = ForaNative.makePythonTupleConverter(
            purePythonModuleImplval.getObjectMember("PyTuple")
            )

        nativeDictConverter = ForaNative.makePythonDictConverter(
            purePythonModuleImplval.getObjectMember("PyDict")
            )

        foraBuiltinsImplVal = ModuleImporter.builtinModuleImplVal()

        return Converter(
            nativeListConverter=nativeListConverter,
            nativeTupleConverter=nativeTupleConverter,
            nativeDictConverter=nativeDictConverter,
            nativeConstantConverter=nativeConstantConverter,
            singletonAndExceptionConverter=singletonAndExceptionConverter,
            vdmOverride=vdm,
            purePythonModuleImplVal=purePythonModuleImplval,
            foraBuiltinsImplVal=foraBuiltinsImplVal
            )
コード例 #20
0
ファイル: CumulusGatewayRemote.py プロジェクト: ufora/ufora
    def tryOnWorkerAdd(self, machineId, ip, ports):
        if self.isTornDown_:
            return False

        guid = HashNative.Hash.sha1(str(uuid.uuid4()))

        try:
            # TODO: get the number of cumulus ports from config
            assert len(ports) == 2

            channels = []
            for i in range(2):
                channel = self.connectToWorker(machineId, ip, ports[i], guid)
                assert channel is not None
                channels.append(channel)

            logging.info("CumulusClient %s successfully connected to both channels of  %s",
                self.cumulusClientId,
                machineId
                )

            with self.lock_:
                if machineId not in self.desiredMachines_:
                    return False


            with self.lock_:
                if machineId in self.disconnectedMachines_:
                    return True

                self.cumulusClient.addMachine(
                    machineId,
                    channels,
                    ModuleImporter.builtinModuleImplVal(),
                    self.callbackScheduler
                    )

                self.connectedMachines_.add(machineId)
                self.desiredMachines_.discard(machineId)

            return True
        except:
            logging.error("Failed: %s", traceback.format_exc())

            return False
コード例 #21
0
    def tryOnWorkerAdd(self, machineId, ip, ports):
        if self.isTornDown_:
            return False

        guid = HashNative.Hash.sha1(str(uuid.uuid4()))

        try:
            # TODO: get the number of cumulus ports from config
            assert len(ports) == 2

            channels = []
            for i in range(2):
                channel = self.connectToWorker(machineId, ip, ports[i], guid)
                assert channel is not None
                channels.append(channel)

            logging.info(
                "CumulusClient %s successfully connected to both channels of  %s",
                self.cumulusClientId, machineId)

            with self.lock_:
                if machineId not in self.desiredMachines_:
                    return False

            with self.lock_:
                if machineId in self.disconnectedMachines_:
                    return True

                self.cumulusClient.addMachine(
                    machineId, channels, ModuleImporter.builtinModuleImplVal(),
                    self.callbackScheduler)

                self.connectedMachines_.add(machineId)
                self.desiredMachines_.discard(machineId)

            return True
        except:
            logging.error("Failed: %s", traceback.format_exc())

            return False
コード例 #22
0
ファイル: FORA.py プロジェクト: Sandy4321/ufora
def reloadBuiltin():
    global _builtin
    ModuleImporter.initialize(reimport = True)
    _builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
コード例 #23
0
ファイル: CumulusService.py プロジェクト: ufora/ufora
    def connectToRemoteWorker(self, machineId, ip, port, guid):
        logging.debug("Attempting to connect to machine %s on %s:%s with guid %s",
                      machineId,
                      ip,
                      port,
                      guid)
        try:
            stringChannel = self._channelFactory.createChannel((ip, port))
        except:
            logging.error("CAN'T CONNECT TO WORKER ON %s:%s!\n"
                          "This may be a temporary failure but if the problem persists, "
                          "check the workers' network configuration and verify "
                          "that the machines can see each other.",
                          ip, port)
            raise

        stringChannel.write(ufora.version)

        stringChannel.write(machineId.__getstate__())

        #initiate the handshake
        stringChannel.write(
            CumulusNative.CumulusClientOrMachine.Machine(
                self.machineId
                ).__getstate__()
            )

        stringChannel.write(guid.__getstate__())

        logging.debug("CumulusService %s wrote handshake for %s with guid %s",
                      self.machineId,
                      machineId,
                      guid)

        channelAsQueue = stringChannel.makeQueuelike(self.callbackScheduler)

        msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)

        if msg is None:
            logging.error("CAN'T CONNECT TO WORKER ON %s:%s!\n"
                          "While attempting to add worker %s with guid %s, "
                          "Worker %s did not receive a builtin hash message "
                          "during handshake.\n"
                          "Verify that the ufora worker is running on the remote machine.",
                          ip, port,
                          machineId, guid,
                          self.machineId)
            return None

        otherWorkersBuiltinHash = Hash.Hash(0)
        otherWorkersBuiltinHash.__setstate__(msg)

        builtinsAgree = otherWorkersBuiltinHash == ModuleImporter.builtinModuleImplVal().hash

        if not builtinsAgree:
            logging.critical("CAN'T CONNECT TO WORKER ON %s:%s!\n"
                             "Worker %s could not connect to Worker %s as they have "
                             "different builtins; former's builtin hash: %s, latter's builtin hash: "
                             "%s\n"
                             "Verify that both machines run the same ufora version.",
                             ip, port,
                             self.machineId,
                             machineId,
                             ModuleImporter.builtinModuleImplVal().hash,
                             otherWorkersBuiltinHash)

            channelAsQueue.disconnect()
            return None

        return channelAsQueue
コード例 #24
0
ファイル: MessageProcessor.py プロジェクト: vishnur/ufora
    def __init__(self, callbackScheduler, sharedStateViewFactory, computedValueGatewayFactory):
        self.lock = threading.Lock()
        self.cacheLoadEvents = {}

        self.resultsById_ = {}
        self.eventsById_ = {}

        logging.info("created a component host")

        self.graph = ComputedGraph.ComputedGraph()

        logging.info("created a ComputedGraph")

        Runtime.initialize()
        logging.info("Runtime initialized")

        ModuleImporter.initialize()
        logging.info("Module importer initialized")

        Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())

        self.incomingObjectCache = IncomingObjectCache()
        self.outgoingObjectCache = OutgoingObjectCache()

        self.VDM = VectorDataManager.constructVDM(callbackScheduler)
        self.VDM.setDropUnreferencedPagesWhenFull(True)
        logging.info("created a VDM")

        logging.info("got shared state view factory: %s", sharedStateViewFactory)

        def initValueGateway():
            with self.graph:
                self.computedValueGateway = computedValueGatewayFactory()
                self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway

        def initSynchronizer():
            self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()

            logging.info("created a SharedStateSynchronizer")

            self.synchronizer.attachView(sharedStateViewFactory.createView())

            logging.info("attached shared state view.")

        simultaneously(initSynchronizer, initValueGateway)

        self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()

        self.outstandingMessagesById = {}
        self.expectedMessageId = 0

        self.messageTypeHandlers = {}

        self.messageTypeHandlers["Read"] = self.handleReadMessage
        self.messageTypeHandlers["Assign"] = self.handleAssignMessage
        self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
        self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
        self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds

        self.pendingObjectQueue = []

        self.subscriptions = Subscriptions.Subscriptions(self.graph, self.computedValueGateway, self.synchronizer)
コード例 #25
0
ファイル: CumulusService.py プロジェクト: ufora/ufora
    def __init__(self,
                 ownAddress,
                 channelListener,
                 channelFactory,
                 eventHandler,
                 callbackScheduler,
                 diagnosticsDir,
                 config,
                 viewFactory,
                 s3InterfaceFactory=None,
                 objectStore=None):
        Stoppable.Stoppable.__init__(self)

        #acquire a machineId randomly, using uuid
        self.machineId = CumulusNative.MachineId(
            Hash.Hash.sha1(str(uuid.uuid4()))
            )

        self.ownAddress = ownAddress
        self.callbackScheduler = callbackScheduler
        self.viewFactory = viewFactory
        self.s3InterfaceFactory = s3InterfaceFactory
        self.objectStore = objectStore
        self.threadsStarted_ = False
        self.connectedMachines = set()
        self.connectingMachines = set()  # machines we are in the process of connecting to
        self.droppedMachineIds = set()
        self.lock = threading.RLock()
        self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
        self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
        self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
        self.cumulusTrackTcmalloc = config.cumulusTrackTcmalloc
        self.eventHandler = eventHandler

        self.reconnectPersistentCacheIndexViewThreads = []

        if config.cumulusDiskCacheStorageSubdirectory is not None:
            self.cumulusDiskCacheWantsDeletionOnTeardown = True
            self.cumulusDiskCacheStorageDir = os.path.join(
                config.cumulusDiskCacheStorageDir,
                config.cumulusDiskCacheStorageSubdirectory
                )
        else:
            self.cumulusDiskCacheWantsDeletionOnTeardown = False
            self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir

        self._stopEvent = threading.Event()

        self._channelListener = channelListener
        assert len(self._channelListener.ports) == 2
        self._channelFactory = channelFactory

        Runtime.initialize()
        ModuleImporter.initialize()

        self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
            self.viewFactory
            )

        self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
            target=self._channelListener.start
            )

        self.vdm = VectorDataManager.constructVDM(
            callbackScheduler,
            self.cumulusVectorRamCacheSizeOverride,
            self.cumulusMaxRamCacheSizeOverride
            )

        if self.cumulusTrackTcmalloc:
            self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()

        self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
            viewFactory.createView(retrySeconds=10.0, numRetries=10),
            callbackScheduler
            )

        self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)

        self.deleteCumulusDiskCacheIfNecessary()

        self.offlineCache = CumulusNative.DiskOfflineCache(
            callbackScheduler,
            self.cumulusDiskCacheStorageDir,
            config.cumulusDiskCacheStorageMB * 1024 * 1024,
            config.cumulusDiskCacheStorageFileCount
            )

        #If the "s3InterfaceFactory" is not in-memory, we use real out of process python.
        #it would be better if this were more explicit
        outOfProcess = self.s3InterfaceFactory is not None and self.s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool

        self.outOfProcessPythonTasks = OutOfProcessPythonTasks.OutOfProcessPythonTasks(outOfProcess=outOfProcess)

        self.vdm.initializeOutOfProcessPythonTasks(self.outOfProcessPythonTasks.nativeTasks)

        checkpointInterval = config.cumulusCheckpointIntervalSeconds
        if checkpointInterval == 0:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
        else:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
                checkpointInterval,
                1024 * 1024
                )

        self.cumulusWorker = self.constructCumlusWorker(
            callbackScheduler,
            CumulusNative.CumulusWorkerConfiguration(
                self.machineId,
                self.cumulusThreadCountOverride,
                checkpointPolicy,
                ExecutionContext.createContextConfiguration(),
                diagnosticsDir or ""
                ),
            self.vdm,
            self.offlineCache,
            eventHandler
            )

        self.datasetLoadService = None
        if self.s3InterfaceFactory:
            externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
                callbackScheduler
                )
            self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
                self.s3InterfaceFactory,
                self.objectStore,
                self.vdm,
                externalDatasetChannel.makeQueuelike(callbackScheduler)
                )

        self.cumulusWorker.startComputations()

        if self.datasetLoadService:
            self.datasetLoadService.startService()
コード例 #26
0
    def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm,
                 **kwds):
        #don't modify callers directly
        kwds = dict(kwds)

        if 's3Service' in kwds:
            s3Service = kwds['s3Service']
            del kwds['s3Service']
        else:
            s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory()

        if 'threadsPerWorker' in kwds:
            threadsPerWorker = kwds['threadsPerWorker']
            del kwds['threadsPerWorker']
        else:
            threadsPerWorker = Setup.config().cumulusServiceThreadCount

        if 'memoryPerWorkerMB' in kwds:
            memoryPerWorkerMB = kwds['memoryPerWorkerMB']
            del kwds['memoryPerWorkerMB']
        else:
            memoryPerWorkerMB = 400

        if 'maxMBPerOutOfProcessPythonTask' in kwds:
            maxBytesPerOutOfProcessPythonTask = kwds[
                'maxMBPerOutOfProcessPythonTask'] * 1024 * 1024
            del kwds['maxMBPerOutOfProcessPythonTask']
        else:
            maxBytesPerOutOfProcessPythonTask = None

        workerCount = 1
        if 'workerCount' in kwds:
            workerCount = kwds['workerCount']
            del kwds['workerCount']

        simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
            workerCount,
            0,
            s3Service=s3Service,
            memoryPerWorkerMB=memoryPerWorkerMB,
            callbackScheduler=callbackScheduler,
            threadsPerWorker=threadsPerWorker,
            maxBytesPerOutOfProcessPythonTask=maxBytesPerOutOfProcessPythonTask,
            **kwds)

        CumulusGateway.CumulusGateway.__init__(
            self, callbackScheduler, vdm, simulation.sharedStateViewFactory)

        self.s3Service = s3Service
        self.simulation = simulation

        self.viewFactory = self.simulation.sharedStateViewFactory

        for workerIx in xrange(workerCount):
            worker = self.simulation.getWorker(workerIx)
            workerVdm = self.simulation.getWorkerVdm(workerIx)

            channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel(
                self.callbackScheduler)
            channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel(
                self.callbackScheduler)

            machineId = workerVdm.getMachineId()

            self.cumulusClient.addMachine(
                machineId, [channel1Client, channel2Client],
                ModuleImporter.builtinModuleImplVal(), self.callbackScheduler)

            worker.addCumulusClient(self.cumulusClientId,
                                    [channel1Worker, channel2Worker],
                                    ModuleImporter.builtinModuleImplVal(),
                                    self.callbackScheduler)

        self.worker = self.simulation.getWorker(0)
        self.workerVdm = self.simulation.getWorkerVdm(0)
        self.loadingService = self.simulation.loadingServices[0]
コード例 #27
0
def reloadBuiltin():
    global _builtin
    ModuleImporter.initialize(reimport=True)
    _builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
コード例 #28
0
    def connectToWorker(self, machineId, ip, port, guid):
        with self.lock_:
            stringChannel = self.channelFactory_.createChannel((ip, port))
            builtinsHash = ModuleImporter.builtinModuleImplVal().hash
            clientId = self.cumulusClientId
            callbackScheduler = self.callbackScheduler

        logging.info("Client %s writing version message '%s' to %s", clientId,
                     ufora.version, machineId)

        stringChannel.write(ufora.version)

        logging.info("Client %s writing client ID message to %s", clientId,
                     machineId)

        stringChannel.write(machineId.__getstate__())

        logging.info("Client %s writing expected machineId message to %s",
                     clientId, machineId)

        stringChannel.write(
            CumulusNative.CumulusClientOrMachine.Client(
                clientId).__getstate__())

        logging.info("Client %s writing guid %s to %s", clientId, guid,
                     machineId)
        stringChannel.write(guid.__getstate__())

        channelAsQueue = stringChannel.makeQueuelike(callbackScheduler)
        msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)
        if msg is None:
            logging.error(
                "While attempting to add worker %s, CumulusClient %s did not "
                + "receive a builtin hash message during handshake", machineId,
                clientId)

        assert msg is not None

        logging.info("Client %s received serialized worker's builtin hash",
                     clientId)

        try:
            workersBuiltinHash = HashNative.Hash(0)
            workersBuiltinHash.__setstate__(msg)
        except:
            logging.info("Client received a bad worker hash: %s of size %s",
                         repr(msg), len(msg))
            raise

        builtinsAgree = workersBuiltinHash == builtinsHash

        if not builtinsAgree:
            logging.critical("Could not connect CumulusClient %s to CumulusWorker %s as they " + \
                             "have different builtins; client's builtin hash: %s, worker's " + \
                             "builtin hash: %s. Disconnecting channel",
                             clientId,
                             machineId,
                             builtinsHash,
                             workersBuiltinHash
                             )

            channelAsQueue.disconnect()
            return None

        return channelAsQueue
コード例 #29
0
ファイル: CumulusService.py プロジェクト: nkhuyu/ufora
    def __init__(self,
                 ownAddress,
                 channelListener,
                 channelFactory,
                 eventHandler,
                 callbackScheduler,
                 diagnosticsDir,
                 config,
                 viewFactory):
        Stoppable.Stoppable.__init__(self)

        #acquire a machineId randomly, using uuid
        self.machineId = CumulusNative.MachineId(
            Hash.Hash.sha1(str(uuid.uuid4()))
            )

        self.ownAddress = ownAddress
        self.callbackScheduler = callbackScheduler
        self.viewFactory = viewFactory
        self.threadsStarted_ = False
        self.connectedMachines = set()
        self.connectingMachines = set()  # machines we are in the process of connecting to
        self.droppedMachineIds = set()
        self.lock = threading.RLock()
        self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
        self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
        self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
        self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc

        self.reconnectPersistentCacheIndexViewThreads = []

        if config.cumulusDiskCacheStorageSubdirectory is not None:
            self.cumulusDiskCacheWantsDeletionOnTeardown = True
            self.cumulusDiskCacheStorageDir = os.path.join(
                config.cumulusDiskCacheStorageDir,
                config.cumulusDiskCacheStorageSubdirectory
                )
        else:
            self.cumulusDiskCacheWantsDeletionOnTeardown = False
            self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir


        logging.info(
            "Creating a CumulusService with ram cache of %s / %s MB and %s threads",
            self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusThreadCountOverride
            )

        self._stopEvent = threading.Event()

        self._channelListener = channelListener
        assert len(self._channelListener.ports) == 2
        self._channelFactory = channelFactory

        Runtime.initialize()
        ModuleImporter.initialize()

        self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
            self.viewFactory
            )

        self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
            target=self._channelListener.start
            )

        self.vdm = VectorDataManager.constructVDM(
            callbackScheduler,
            self.cumulusVectorRamCacheSizeOverride,
            self.cumulusMaxRamCacheSizeOverride
            )

        if self.cumulusTrackTcMalloc:
            logging.info(
                "CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
                self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0
                )
            self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()

        self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
            viewFactory.createView(retrySeconds=10.0, numRetries=10),
            callbackScheduler
            )

        self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)

        self.deleteCumulusDiskCacheIfNecessary()

        self.offlineCache = CumulusNative.DiskOfflineCache(
            callbackScheduler,
            self.cumulusDiskCacheStorageDir,
            config.cumulusDiskCacheStorageMB * 1024 * 1024,
            config.cumulusDiskCacheStorageFileCount
            )

        checkpointInterval = config.cumulusCheckpointIntervalSeconds
        if checkpointInterval == 0:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
        else:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
                checkpointInterval,
                1024 * 1024
                )

        self.cumulusWorker = self.constructCumlusWorker(
            callbackScheduler,
            CumulusNative.CumulusWorkerConfiguration(
                self.machineId,
                self.cumulusThreadCountOverride,
                checkpointPolicy,
                ExecutionContext.createContextConfiguration(),
                diagnosticsDir or ""
                ),
            self.vdm,
            self.offlineCache,
            eventHandler
            )

        #externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
            #callbackScheduler
            #)
        #self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
            #settings.s3InterfaceFactory,
            #settings.objectStore,
            #self.vdm,
            #externalDatasetChannel.makeQueuelike(callbackScheduler)
            #)

        self.cumulusWorker.startComputations()
コード例 #30
0
    def initialize(self, purePythonMDSAsJson):
        """Initialize the converter assuming a set of pyfora builtins"""
        try:
            import pyfora.ObjectRegistry as ObjectRegistry
            import ufora.FORA.python.PurePython.Converter as Converter
            import ufora.FORA.python.PurePython.PyforaSingletonAndExceptionConverter as PyforaSingletonAndExceptionConverter
            import ufora.native.FORA as ForaNative
            import ufora.FORA.python.ModuleImporter as ModuleImporter


            logging.info("Initializing the PyforaObjectConverter")

            objectRegistry_[0] = ObjectRegistry.ObjectRegistry()

            if purePythonMDSAsJson is None:
                converter_[0] = Converter.Converter()
            else:
                purePythonModuleImplval = ModuleImporter.importModuleFromMDS(
                    ModuleDirectoryStructure.ModuleDirectoryStructure.fromJson(purePythonMDSAsJson),
                    "fora",
                    "purePython",
                    searchForFreeVariables=True
                    )

                singletonAndExceptionConverter = \
                    PyforaSingletonAndExceptionConverter.PyforaSingletonAndExceptionConverter(
                        purePythonModuleImplval
                        )

                primitiveTypeMapping = {
                    bool: purePythonModuleImplval.getObjectMember("PyBool"),
                    str: purePythonModuleImplval.getObjectMember("PyString"),
                    int: purePythonModuleImplval.getObjectMember("PyInt"),
                    float: purePythonModuleImplval.getObjectMember("PyFloat"),
                    type(None): purePythonModuleImplval.getObjectMember("PyNone"),
                    }


                nativeConstantConverter = ForaNative.PythonConstantConverter(
                    primitiveTypeMapping
                    )

                nativeListConverter = ForaNative.makePythonListConverter(
                    purePythonModuleImplval.getObjectMember("PyList")
                    )

                nativeTupleConverter = ForaNative.makePythonTupleConverter(
                    purePythonModuleImplval.getObjectMember("PyTuple")
                    )

                nativeDictConverter = ForaNative.makePythonDictConverter(
                    purePythonModuleImplval.getObjectMember("PyDict")
                    )

                foraBuiltinsImplVal = ModuleImporter.builtinModuleImplVal()

                converter_[0] = Converter.Converter(
                    nativeListConverter=nativeListConverter,
                    nativeTupleConverter=nativeTupleConverter,
                    nativeDictConverter=nativeDictConverter,
                    nativeConstantConverter=nativeConstantConverter,
                    singletonAndExceptionConverter=singletonAndExceptionConverter,
                    vdmOverride=ComputedValueGateway.getGateway().vdm,
                    purePythonModuleImplVal=purePythonModuleImplval,
                    foraBuiltinsImplVal=foraBuiltinsImplVal
                    )
        except:
            logging.critical("Failed to initialize the PyforaObjectConverter: %s", traceback.format_exc())
            raise
コード例 #31
0
ファイル: CumulusService.py プロジェクト: ufora/ufora
    def connectToRemoteWorker(self, machineId, ip, port, guid):
        logging.debug("Attempting to connect to machine %s on %s:%s with guid %s",
                      machineId,
                      ip,
                      port,
                      guid)
        try:
            stringChannel = self._channelFactory.createChannel((ip, port))
        except:
            logging.error("CAN'T CONNECT TO WORKER ON %s:%s!\n"
                          "This may be a temporary failure but if the problem persists, "
                          "check the workers' network configuration and verify "
                          "that the machines can see each other.",
                          ip, port)
            raise

        stringChannel.write(ufora.version)

        stringChannel.write(machineId.__getstate__())

        #initiate the handshake
        stringChannel.write(
            CumulusNative.CumulusClientOrMachine.Machine(
                self.machineId
                ).__getstate__()
            )

        stringChannel.write(guid.__getstate__())

        logging.debug("CumulusService %s wrote handshake for %s with guid %s",
                      self.machineId,
                      machineId,
                      guid)

        channelAsQueue = stringChannel.makeQueuelike(self.callbackScheduler)

        msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)

        if msg is None:
            logging.error("CAN'T CONNECT TO WORKER ON %s:%s!\n"
                          "While attempting to add worker %s with guid %s, "
                          "Worker %s did not receive a builtin hash message "
                          "during handshake.\n"
                          "Verify that the ufora worker is running on the remote machine.",
                          ip, port,
                          machineId, guid,
                          self.machineId)
            return None

        otherWorkersBuiltinHash = Hash.Hash(0)
        otherWorkersBuiltinHash.__setstate__(msg)

        builtinsAgree = otherWorkersBuiltinHash == ModuleImporter.builtinModuleImplVal().hash

        if not builtinsAgree:
            logging.critical("CAN'T CONNECT TO WORKER ON %s:%s!\n"
                             "Worker %s could not connect to Worker %s as they have "
                             "different builtins; former's builtin hash: %s, latter's builtin hash: "
                             "%s\n"
                             "Verify that both machines run the same ufora version.",
                             ip, port,
                             self.machineId,
                             machineId,
                             ModuleImporter.builtinModuleImplVal().hash,
                             otherWorkersBuiltinHash)

            channelAsQueue.disconnect()
            return None

        return channelAsQueue
コード例 #32
0
ファイル: DefaultConverters.py プロジェクト: Sandy4321/ufora
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import os

import ufora.FORA.python.ModuleImporter as ModuleImporter

import pyfora

import ufora.native.FORA as ForaNative

builtinPythonImplVal = ModuleImporter.importModuleFromPath(
    os.path.join(os.path.abspath(os.path.split(pyfora.__file__)[0]), "fora", "purePython"),
    searchForFreeVariables=True
    )

primitiveTypeMapping = {
    bool: builtinPythonImplVal.getObjectMember("PyBool"),
    str: builtinPythonImplVal.getObjectMember("PyString"),
    int: builtinPythonImplVal.getObjectMember("PyInt"),
    float: builtinPythonImplVal.getObjectMember("PyFloat"),
    type(None): builtinPythonImplVal.getObjectMember("PyNone"),
    }

defaultWrappingNativeListConverter = ForaNative.makePythonListConverter(
    builtinPythonImplVal.getObjectMember("PyList")
    )

defaultWrappingNativeTupleConverter = ForaNative.makePythonTupleConverter(
コード例 #33
0
ファイル: CumulusService.py プロジェクト: ufora/ufora
    def doChannelHandshake(self, channel):
        try:
            logging.debug("Worker %s beginning channel handshake", self.machineId)
            version = channel.getTimeout(HANDSHAKE_TIMEOUT)
            if version is None:
                logging.error(
                    "CAN'T ACCEPT CONNECTION!\n"
                    "CumulusService %s couldn't read client version within the configured timeout",
                    self.machineId
                    )

            if version != ufora.version:
                self.logBadUforaVersionOnChannel(version)
                channel.disconnect()
                return

            logging.debug(
                "CumulusService %s accepted connection from client with version %s",
                self.machineId,
                version
                )

            msgThatShouldBeMyOwnHash = channel.getTimeout(HANDSHAKE_TIMEOUT)
            if not self.isOwnHashInHandshakeMessage(msgThatShouldBeMyOwnHash):
                channel.disconnect()
                return

            msg = channel.getTimeout(HANDSHAKE_TIMEOUT)
            if msg is None:
                logging.error(
                    "CAN'T ACCEPT CONNECTION!\n"
                    "Worker %s didn't received remote machine ID during handshake",
                    self.machineId
                    )
                channel.disconnect()
                return

            clientOrMachine = CumulusNative.CumulusClientOrMachine.Machine(
                CumulusNative.MachineId(
                    Hash.Hash(0)
                    )
                )
            clientOrMachine.__setstate__(msg)

            hashGuid = Hash.Hash(0)
            msg = channel.getTimeout(HANDSHAKE_TIMEOUT)
            if msg is None:
                logging.error(
                    "CAN'T ACCEPT CONNECTION!\n"
                    "Worker %s didn't received handshake GUID",
                    self.machineId
                    )
                channel.disconnect()
                return

            hashGuid.__setstate__(msg)
            logging.debug(
                "Worker %s accepted connection with guid %s from %s",
                self.machineId,
                hashGuid,
                clientOrMachine
                )

            channel.write(
                ModuleImporter.builtinModuleImplVal().hash.__getstate__()
                )

            with self.lock:
                self._channelListener.setGroupIdForAcceptedChannel(
                    channel,
                    (clientOrMachine, hashGuid)
                    )

            logging.debug("CumulusService %s added a channel to group %s",
                          self.machineId,
                          (clientOrMachine, hashGuid))
        except:
            logging.error("FAILED TO PROCESS INCOMING CONNECTION: %s", traceback.format_exc())
            channel.disconnect()
コード例 #34
0
#   Unless required by applicable law or agreed to in writing, software
#   distributed under the License is distributed on an "AS IS" BASIS,
#   WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
#   See the License for the specific language governing permissions and
#   limitations under the License.

import os

import ufora.FORA.python.ModuleImporter as ModuleImporter

import pyfora

import ufora.native.FORA as ForaNative

builtinPythonImplVal = ModuleImporter.importModuleFromPath(
    os.path.join(os.path.abspath(os.path.split(pyfora.__file__)[0]), "fora",
                 "purePython"),
    searchForFreeVariables=True)

primitiveTypeMapping = {
    bool: builtinPythonImplVal.getObjectMember("PyBool"),
    str: builtinPythonImplVal.getObjectMember("PyString"),
    int: builtinPythonImplVal.getObjectMember("PyInt"),
    float: builtinPythonImplVal.getObjectMember("PyFloat"),
    type(None): builtinPythonImplVal.getObjectMember("PyNone"),
}

defaultWrappingNativeListConverter = ForaNative.makePythonListConverter(
    builtinPythonImplVal.getObjectMember("PyList"))

defaultWrappingNativeTupleConverter = ForaNative.makePythonTupleConverter(
    builtinPythonImplVal.getObjectMember("PyTuple"))
コード例 #35
0
    def __init__(self, ownAddress, channelListener, channelFactory,
                 eventHandler, callbackScheduler, diagnosticsDir, config,
                 viewFactory):
        Stoppable.Stoppable.__init__(self)

        #acquire a machineId randomly, using uuid
        self.machineId = CumulusNative.MachineId(
            Hash.Hash.sha1(str(uuid.uuid4())))

        self.ownAddress = ownAddress
        self.callbackScheduler = callbackScheduler
        self.viewFactory = viewFactory
        self.threadsStarted_ = False
        self.connectedMachines = set()
        self.connectingMachines = set(
        )  # machines we are in the process of connecting to
        self.droppedMachineIds = set()
        self.lock = threading.RLock()
        self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024 * 1024
        self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024 * 1024
        self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
        self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc

        self.reconnectPersistentCacheIndexViewThreads = []

        if config.cumulusDiskCacheStorageSubdirectory is not None:
            self.cumulusDiskCacheWantsDeletionOnTeardown = True
            self.cumulusDiskCacheStorageDir = os.path.join(
                config.cumulusDiskCacheStorageDir,
                config.cumulusDiskCacheStorageSubdirectory)
        else:
            self.cumulusDiskCacheWantsDeletionOnTeardown = False
            self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir

        logging.info(
            "Creating a CumulusService with ram cache of %s / %s MB and %s threads",
            self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
            self.cumulusThreadCountOverride)

        self._stopEvent = threading.Event()

        self._channelListener = channelListener
        assert len(self._channelListener.ports) == 2
        self._channelFactory = channelFactory

        Runtime.initialize()
        ModuleImporter.initialize()

        self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
            self.viewFactory)

        self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
            target=self._channelListener.start)

        self.vdm = VectorDataManager.constructVDM(
            callbackScheduler, self.cumulusVectorRamCacheSizeOverride,
            self.cumulusMaxRamCacheSizeOverride)

        if self.cumulusTrackTcMalloc:
            logging.info(
                "CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
                self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0)
            self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()

        self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
            viewFactory.createView(retrySeconds=10.0, numRetries=10),
            callbackScheduler)

        self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)

        self.deleteCumulusDiskCacheIfNecessary()

        self.offlineCache = CumulusNative.DiskOfflineCache(
            callbackScheduler, self.cumulusDiskCacheStorageDir,
            config.cumulusDiskCacheStorageMB * 1024 * 1024,
            config.cumulusDiskCacheStorageFileCount)

        checkpointInterval = config.cumulusCheckpointIntervalSeconds
        if checkpointInterval == 0:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None ()
        else:
            checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
                checkpointInterval, 1024 * 1024)

        self.cumulusWorker = self.constructCumlusWorker(
            callbackScheduler,
            CumulusNative.CumulusWorkerConfiguration(
                self.machineId,
                self.cumulusThreadCountOverride, checkpointPolicy,
                ExecutionContext.createContextConfiguration(), diagnosticsDir
                or ""), self.vdm, self.offlineCache, eventHandler)

        #externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
        #callbackScheduler
        #)
        #self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
        #settings.s3InterfaceFactory,
        #settings.objectStore,
        #self.vdm,
        #externalDatasetChannel.makeQueuelike(callbackScheduler)
        #)

        self.cumulusWorker.startComputations()
コード例 #36
0
ファイル: CumulusService.py プロジェクト: nkhuyu/ufora
    def doChannelHandshake(self, channel):
        try:
            logging.info("CumulusService %s beginning channel handshake on channel %s", self.machineId, channel)
            version = channel.getTimeout(HANDSHAKE_TIMEOUT)
            if version is None:
                logging.error(
                    "CumulusService %s couldn't read client version within the configured timeout",
                    self.machineId
                    )

            if version != ufora.version:
                self.logBadUforaVersionOnChannel(version)
                channel.disconnect()
                return

            logging.info(
                "CumulusService %s accepted connection from client with version %s",
                self.machineId,
                version
                )

            msgThatShouldBeMyOwnHash = channel.getTimeout(HANDSHAKE_TIMEOUT)
            if not self.isOwnHashInHandshakeMessage(msgThatShouldBeMyOwnHash):
                channel.disconnect()
                return

            msg = channel.getTimeout(HANDSHAKE_TIMEOUT)
            if msg is None:
                logging.error(
                    "CumulusService %s didn't received remote machine Id in handshake",
                    self.machineId
                    )
                channel.disconnect()
                return

            clientOrMachine = CumulusNative.CumulusClientOrMachine.Machine(
                CumulusNative.MachineId(
                    Hash.Hash(0)
                    )
                )
            clientOrMachine.__setstate__(msg)

            hashGuid = Hash.Hash(0)
            msg = channel.getTimeout(HANDSHAKE_TIMEOUT)
            if msg is None:
                logging.error("CumulusService %s didn't received handshake GUID", self.machineId)
                channel.disconnect()
                return

            hashGuid.__setstate__(msg)
            logging.info(
                "CumulusService %s accepted connection of guid %s for client %s",
                self.machineId,
                hashGuid,
                clientOrMachine
                )

            channel.write(
                ModuleImporter.builtinModuleImplVal().hash.__getstate__()
                )

            with self.lock:
                self._channelListener.setGroupIdForAcceptedChannel(
                    channel,
                    (clientOrMachine, hashGuid)
                    )

            logging.info("CumulusService %s added a channel to group %s",
                         self.machineId,
                         (clientOrMachine, hashGuid))
        except:
            logging.error("Failed to process incoming connection: %s", traceback.format_exc())
            channel.disconnect()
コード例 #37
0
ファイル: MessageProcessor.py プロジェクト: vishnur/ufora
    def __init__(self,
                 callbackScheduler,
                 sharedStateViewFactory,
                 computedValueGatewayFactory):
        self.lock = threading.Lock()
        self.cacheLoadEvents = {}

        self.resultsById_ = {}
        self.eventsById_ = {}

        logging.info("created a component host")

        self.graph = ComputedGraph.ComputedGraph()

        logging.info("created a ComputedGraph")

        Runtime.initialize()
        logging.info("Runtime initialized")

        ModuleImporter.initialize()
        logging.info("Module importer initialized")


        Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())

        self.incomingObjectCache = IncomingObjectCache()
        self.outgoingObjectCache = OutgoingObjectCache()

        self.VDM = VectorDataManager.constructVDM(callbackScheduler)
        self.VDM.setDropUnreferencedPagesWhenFull(True)
        logging.info("created a VDM")

        logging.info("got shared state view factory: %s", sharedStateViewFactory)

        def initValueGateway():
            with self.graph:
                self.computedValueGateway = computedValueGatewayFactory()
                self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway


        def initSynchronizer():
            self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()

            logging.info("created a SharedStateSynchronizer")

            self.synchronizer.attachView(
                sharedStateViewFactory.createView()
                )

            logging.info("attached shared state view.")

        simultaneously(
            initSynchronizer,
            initValueGateway
            )

        self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()

        self.outstandingMessagesById = {}
        self.expectedMessageId = 0

        self.messageTypeHandlers = {}

        self.messageTypeHandlers["Read"] = self.handleReadMessage
        self.messageTypeHandlers["Assign"] = self.handleAssignMessage
        self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
        self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
        self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds

        self.pendingObjectQueue = []

        self.subscriptions = Subscriptions.Subscriptions(
            self.graph,
            self.computedValueGateway,
            self.synchronizer
            )