def start(self): assert not self.hasStarted if self.actuallyRunOutOfProcess: def onStdout(msg): logging.info("OutOfProcessDownloader Out> %s", msg) def onStderr(msg): logging.info("OutOfProcessDownloader Err> %s", msg) self.childSubprocess = SubprocessRunner.SubprocessRunner( [sys.executable, __file__, str(self.childSocket.fileno())], onStdout, onStderr) self.childSubprocess.start() self.hasStarted = True self.backgroundThread = ManagedThread.ManagedThread( target=self.watchChild_) self.backgroundThread.start() else: self.hasStarted = True self.backgroundThread = ManagedThread.ManagedThread( target=self.executeChild_) self.backgroundThread.start()
def __init__(self, callbackScheduler, vdm, viewFactory): Stoppable.Stoppable.__init__(self) self.lock_ = threading.Lock() self.callbackScheduler = callbackScheduler self.definitionToIdMap_ = {} self.idToDefinitionMap_ = {} self.vdm = vdm self.onJsonViewOfSystemChanged = None self.persistentCacheIndex = CumulusNative.PersistentCacheIndex( viewFactory.createView(retrySeconds=10.0, numRetries=10), callbackScheduler) self.vdm.setPersistentCacheIndex(self.persistentCacheIndex) self.cumulusClientId = CumulusNative.CumulusClientId( Hash.Hash.sha1(str(uuid.uuid4()))) logging.info("CumulusClient created with %s", self.cumulusClientId) self.cumulusClient = CumulusNative.CumulusClient( vdm, self.cumulusClientId, self.callbackScheduler) self.finalResponses = Queue.Queue() self.cumulusClientListener = self.cumulusClient.createListener() self.cpuAssignmentDependencyGraph = CumulusNative.CpuAssignmentDependencyGraph( self.callbackScheduler.getFactory().createScheduler( self.callbackScheduler.getMetadata() + "_cpuAssignmentGraph", 1), self.vdm) self.cpuAssignmentDependencyGraph.subscribeToCumulusClient( self.cumulusClient) self.pendingCallbacksByGuid = {} self.cpuAssignmentDependencyGraphListener = \ self.cpuAssignmentDependencyGraph.createListener() self.threads = [] self.threads.append( ManagedThread.ManagedThread(target=self.processClientMessages_, args=())) self.threads.append( ManagedThread.ManagedThread( target=self.processDependencyGraphMessages_, args=())) for t in self.threads: t.start() self.nextCpuUpdateTime = time.time() self.cpuMessagesSinceLastUpdate = 0 self.lastSystemwideUpdateTime = time.time()
def onConnectionAvailable(self, channels, clientOrMachineAndGuid): def performConnect(): clientOrMachine, guid = clientOrMachineAndGuid logging.info("Connection is available to %s with guid %s", clientOrMachine, guid) with self.lock: if self.shouldStop(): logging.info( "Rejecting a connection because we are no longer active" ) for channel in channels: channel.disconnect() return if clientOrMachine.isMachine(): with self.lock: if clientOrMachine.asMachine.machine in self.droppedMachineIds: return self.connectedMachines.add( clientOrMachine.asMachine.machine) self.cumulusWorker.addMachine( clientOrMachine.asMachine.machine, channels, ModuleImporter.builtinModuleImplVal(), self.callbackScheduler) else: self.cumulusWorker.addCumulusClient( clientOrMachine.asClient.client, channels, ModuleImporter.builtinModuleImplVal(), self.callbackScheduler) ManagedThread.ManagedThread(target=performConnect, args=()).start()
def onReconnectedToSharedState(self): thread = ManagedThread.ManagedThread( target=self.updatePersistentCacheView, args=()) self.reconnectPersistentCacheIndexViewThreads.append(thread) thread.start()
def __init__(self, inMemoryCluster): self.inMemoryCluster = inMemoryCluster self._onClusterResponse = None self.desires = {} self.callbackQueue = Queue.Queue() self.callbackThread = ManagedThread.ManagedThread(target=self.callbackLoop)
def onWorkerAdd(self, ip, ports, machineIdAsString): machineId = CumulusNative.MachineId(Hash.Hash.stringToHash(machineIdAsString)) if machineId <= self.machineId: logging.info("Worker %s detected worker %s, and waiting for incoming connection", self.machineId, machineId) #only connect one way. If the worker is larger than us, then we connect to it return guid = Hash.Hash.sha1(str(uuid.uuid4())) logging.info( "Worker %s detected worker %s and initiating connection with guid %s", self.machineId, machineId, guid ) with self.lock: # Track that we are trying to connect to this machine self.connectingMachines.add(machineId) ManagedThread.ManagedThread( target=self.onWorkerAdd2, args=(machineId, ip, ports, guid) ).start()
def setDisconnected(self): logging.warn("SharedState is disconnected from CAM. Trying to reconnect.") with self._lock: self._lastDisconnectTime = time.time() self._lastReconnectTime = None self.isConnected = False def sleepAndTriggerUpdate(): while (not self.shouldStop() and ( self._lastReconnectTime is None or time.time() <= self._lastReconnectTime + TIME_TO_SLEEP_AFTER_RECONNECT + 0.01) ): time.sleep(0.01) self.updateActiveMachinesOnReactorThread() newThread = ManagedThread.ManagedThread(target = sleepAndTriggerUpdate) with self._lock: self._triggerUpdateAfterDisconnectThreads.append(newThread) newThread.start() if self.isConnecting: logging.warn("CAM disconnected from SharedState while connecting. Setting a flag and sleeping.") self.disconnectedWhileConnecting = True connectDirectly = False else: connectDirectly = True if connectDirectly: self.connectView()
def __init__(self, vdm, offlineCache): Stoppable.Stoppable.__init__(self) self.dependencies_ = TwoWaySetMap.TwoWaySetMap() self.vdm_ = vdm self.offlineCache_ = offlineCache self.finishedValuesAndTimeElapsed_ = {} self.intermediates_ = {} self.lock_ = threading.RLock() self.completable_ = Queue.Queue() self.timesComputed = 0 self.computingContexts_ = {} self.computingContexts_t0_ = {} self.isSplit_ = set() self.watchers_ = {} self.contexts_ = [] self.inProcessDownloader = ( OutOfProcessDownloader.OutOfProcessDownloaderPool( Setup.config().cumulusServiceThreadCount, actuallyRunOutOfProcess = False ) ) self.threads_ = [] self.isActive = True #setup the primary cache object, and set its worker threads going for threadIx in range(Setup.config().cumulusServiceThreadCount): workerThread = ManagedThread.ManagedThread(target = self.threadWorker) workerThread.start() self.threads_.append(workerThread)
def test_filtered_channel(self): """Verify that message filtering works The sender sends a message that is expected to be filtered by FilteredChannel. The test verifies that the receiver doesn't get the message """ channelFilter = FilteredChannelFactory.FilteredChannel( callbackScheduler, self.filterMinId) try: sender, receiver = channelFilter.getChannelPair() filteredMessage = SharedState.MessageOut.MinimumIdResponse(0) sender.write(filteredMessage) toGet = [None] def tryGet(): try: toGet[0] = receiver.get() except UserWarning: if channelFilter.stopFlag.is_set(): return else: raise t = ManagedThread.ManagedThread(target=tryGet) t.start() t.join(1) self.assertIsNone(toGet[0]) finally: channelFilter.teardown()
def __init__(self, viewFactory): Stoppable.Stoppable.__init__(self) self.viewFactory = viewFactory self.activeMachineIds = set() self._outgoingNotifications = Queue.Queue() self._lock = threading.RLock() self._lastDisconnectTime = None self._lastReconnectTime = None self._registeredIpPortAndMachineId = None logging.debug("Connecting to shared state: %s", viewFactory) self.listeners_ = set() self.clientID = None self.isConnected = False self.isConnecting = False self.disconnectedWhileConnecting = False self._triggerUpdateAfterDisconnectThreads = [] self.clientIdToIpPortAndMachineIdAsString = {} self.machineIdToClientId = {} self.workerStatusKeyspace = SharedState.Keyspace( "ComparisonKeyType", NativeJson.Json( (('P', 'CumulusNodeStatus'), )), 1) self.asyncView = None self.eventLoopThread = ManagedThread.ManagedThread( target=self.eventDispatchLoop)
def test_shared_state_hang_on_view_id(self): try: def createFilteredChannelFactory(callbackScheduler, manager): return FilteredChannelFactory.FilteredChannelFactory( callbackScheduler, manager, self.filterInitialize) harness = SharedStateTestHarness.SharedStateTestHarness( True, inMemChannelFactoryFactory=createFilteredChannelFactory) view = harness.viewFactory.createView() def getViewId(): try: view.id except UserWarning: pass thread = ManagedThread.ManagedThread(target=getViewId) thread.start() thread.join(1) self.assertTrue(thread.isAlive()) view.teardown() thread.join() finally: harness.teardown()
def _channelConnectCallback(self, channel): channel = channel.makeQueuelike(callbackScheduler) t = ManagedThread.ManagedThread(target=self._echoLoop, args=(channel, )) t.start() self.threads.append(t) self.channels.append(t)
def simultaneously(*factories): threads = [ManagedThread.ManagedThread(target=x) for x in factories] for t in threads: t.start() for t in threads: t.join()
def startLoop(logfileLocation, interval = 1.0): global started if started: return started = True t = ManagedThread.ManagedThread(target = memoryWriteLoop, args = (logfileLocation,interval)) t.daemon = True t.start()
def stressMultipleSharedStateReadWrites(self, useTcpFactory=False, keysToWrite=20, threadcount=10): keyspaceSize = keysToWrite * 5 subPasses = 10 if useTcpFactory: viewFactory = ViewFactory.ViewFactory.TcpViewFactory( callbackScheduler, address="localhost") else: viewFactory = self.simulator.getViewFactory() worked = {} for ix in range(threadcount): worked[ix] = False def test(threadIx): for subPassIx in range(subPasses): logging.info("Thread %s starting pass %s", threadIx, subPassIx) testKeyspace = SharedState.Keyspace( "TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1) view = viewFactory.createView() rng = SharedState.KeyRange(testKeyspace, 0, None, None, True, False) view.subscribe(rng) for ix in range(keysToWrite): with SharedState.Transaction(view): ix = random.randint(0, keyspaceSize) key = SharedState.Key( testKeyspace, (NativeJson.Json("key %s" % ix), )) value = uuid.uuid4().hex view[key] = NativeJson.Json(value) worked[threadIx] = True threads = [ ManagedThread.ManagedThread(target=test, args=(ix, )) for ix in range(threadcount) ] for t in threads: t.start() for t in threads: t.join() expectedDict = {} for ix in range(threadcount): expectedDict[ix] = True self.assertEqual(expectedDict, worked)
def __init__(self, numKeys, keySize): self.q = Queue.Queue() self.port = Setup.config().testPort self.server = SimpleServer.SimpleServer(self.port) self.server._onConnect = lambda sock, address : self.q.put(sock) self.serverThread = ManagedThread.ManagedThread(target=self.server.start) self.serverThread.daemon = True self.serverThread.start() ChannelTester.__init__(self, self.createSocketChannels, numKeys, keySize)
def startService(self): with self.lock_: if self.threads_: return logging.debug("Starting %s PythonIoTasks service threads", Setup.config().externalDatasetLoaderServiceThreads) for ix in range(self.threadcount): self.threads_.append( ManagedThread.ManagedThread(target=self.loadLoop)) self.threads_[-1].start()
def __init__(self, callbackScheduler, filterFun): self.filterFun = filterFun self.callbackScheduler = callbackScheduler self.viewFacingViewChannel, self.filterFacingManagerChannel = SharedStateNative.InMemoryChannel( self.callbackScheduler) self.filterFacingViewChannel, self.managerFacingManagerChannel = SharedStateNative.InMemoryChannel( self.callbackScheduler) self.stopFlag = threading.Event() self.channelPumpThreads = [] self.channelPumpThreads.append( ManagedThread.ManagedThread(target=self.filteredChannelPump, args=(self.filterFacingManagerChannel, self.filterFacingViewChannel))) self.channelPumpThreads.append( ManagedThread.ManagedThread( target=self.filteredChannelPump, args=(self.filterFacingViewChannel, self.filterFacingManagerChannel))) for thread in self.channelPumpThreads: thread.start()
def onChannelConnect(self, portIndex, cumulusChannel): logging.debug("Received incoming connection on port id %d", portIndex) with self.lock: if self.shouldStop(): logging.info("Rejecting a connection because we are no longer active") self._channelListener.rejectIncomingChannel(cumulusChannel) return # We process the incoming channel in a separate thread to minimize the # amount of processing done in the socket-accept thread. ManagedThread.ManagedThread( target=self.doChannelHandshake, args=(cumulusChannel,) ).start()
def setUp(self): logging.info("TestSocketChannel setting up") self.port = Setup.config().testPort self.serverChannel = None self.server = None self.serverChannelEvent = threading.Event() self.tries = 0 self.server = TestServer(self.port, self.onConnect) self.serverThread = ManagedThread.ManagedThread( target=self.server.start) self.serverThread.start() self.server.blockUntilListening() logging.info("Server thread is listening")
def __init__(self, callbackScheduler, cachePathOverride=None, port=None): self.callbackScheduler = callbackScheduler port = Setup.config().sharedStatePort logging.info("Initializing SharedStateService with port = %s", port) self.cachePath = cachePathOverride if cachePathOverride is not None else \ Setup.config().sharedStateCache if self.cachePath != '' and not os.path.exists(self.cachePath): os.makedirs(self.cachePath) CloudService.Service.__init__(self) self.socketServer = SimpleServer.SimpleServer(port) self.keyspaceManager = KeyspaceManager( 0, 1, pingInterval=120, cachePathOverride=cachePathOverride) self.socketServer._onConnect = self.onConnect self.socketServerThread = ManagedThread.ManagedThread( target=self.socketServer.start) self.logfilePruneThread = ManagedThread.ManagedThread( target=self.logFilePruner) self.stoppedFlag = threading.Event()
def __init__(self, port, portScanIncrement=0): assert isinstance(port, int) assert isinstance(portScanIncrement, int) self.requestedPort = port self.port = port self.portScanIncrement = portScanIncrement self.socketServer = SimpleServer.SimpleServer(self.port) self.boundEvent_ = threading.Event() self.tornDown = False self.onPortBound = None self.socketConnectCallback = None self.socketServer._onConnect = self.onSocketConnect self.listenerThread = ManagedThread.ManagedThread( target=self.socketServer.start)
def __init__(self, callbackScheduler, channelListener, sharedStateAddress): Stoppable.Stoppable.__init__(self) self._lock = threading.Lock() self.callbackScheduler = callbackScheduler self.sharedStateAddress = sharedStateAddress self.channelListener = channelListener self.channelListener.registerConnectCallback( self.onSubscribableConnection) ModuleImporter.initialize() self.socketsToDisconnectOnExit = [] self.procsToKillOnExit = set() self.isTornDown_ = False self.cleanupThread = ManagedThread.ManagedThread( target=self.cleanupThread_)
def test_socket_channel_shutdown(self): done = threading.Event() listener = ChannelListener.SocketListener(self.port) listener.registerConnectCallback(lambda sock, address: done.set()) try: thread = ManagedThread.ManagedThread(target=listener.start) thread.start() listener.blockUntilReady() TcpChannelFactory.TcpStringChannelFactory( callbackScheduler).createChannel(('localhost', self.port)) self.assertTrue(done.wait(2)) finally: listener.stop() thread.join()
def onWorkerAdd(self, ip, ports, machineIdAsString): machineId = CumulusNative.MachineId( HashNative.Hash.stringToHash(machineIdAsString)) with self.lock_: if self.isTornDown_: return logging.info("CumulusClient %s preparing to connect to %s", self.cumulusClientId, machineId) self.desiredMachines_.add(machineId) newThread = ManagedThread.ManagedThread( target=self.addDesiredMachine, args=(machineId, ip, ports)) self.connectingThreads_.append(newThread) self.connectingThreads_ = [ x for x in self.connectingThreads_ if x.isAlive() ] newThread.start()
def __init__(self, viewFactory, onConnectCallback=lambda result: None, onErrorCallback=lambda error: None): self._stopFlag = threading.Event() self._onConnectCallback = onConnectCallback self._onErrorCallback = onErrorCallback self._view = viewFactory.createView(retrySeconds=10.0, numRetries=10) self._listener = SharedState.Listener(self._view) self._subscribedKeyspaces = set() self._keyspaceCallbackFunctions = {} self._subscriptionDeferreds = {} self._reactorThreadCalls = Queue.Queue() self._viewReactorThread = ManagedThread.ManagedThread( target=self._viewReactorLoop)
def _startChannelLoop(self, channelId, channel): def channelLoop(): while not self.shouldStop: try: msg = channel.getTimeout(.1) except UserWarning: #we've been disconnected. propagate the disconnect and exit self.onMessageReceived_("", channelId) return except: self.onMessageReceived_("", channelId) raise if msg is not None: self.onMessageReceived_(msg, channelId) channel.disconnect() self.channelThreads[channelId] = ManagedThread.ManagedThread( target=channelLoop) self.channelThreads[channelId].start()
def test_filtered_channels_noblock(self): channelFilter = FilteredChannelFactory.FilteredChannel( callbackScheduler, self.filterAllButMinId) try: viewChannel, managerChannel = channelFilter.getChannelPair() MessageOut = getattr(SharedStateNative, 'SharedState::MessageOut') filteredMessage = MessageOut.MinimumIdResponse(0) allowedMessage = MessageOut.FlushRequest(0) viewChannel.write(filteredMessage) viewChannel.write(allowedMessage) toGet = [None] def tryGet(): toGet[0] = managerChannel.get() t = ManagedThread.ManagedThread(target=tryGet) t.start() t.join(4) self.assertIsNotNone(toGet[0]) finally: channelFilter.teardown()
def test_socket_listener(self): server = ChannelEchoServer(self.port) try: thread = ManagedThread.ManagedThread(target=server.start) thread.start() server.blockUntilReady() stringChannelFactory = TcpChannelFactory.TcpStringChannelFactory( callbackScheduler) channel = stringChannelFactory.createChannel( ('localhost', self.port)) channel = channel.makeQueuelike(callbackScheduler) toSend = "Hi There!" channel.write(toSend) self.assertEquals(toSend, channel.get()) finally: try: server.teardown() except UserWarning: pass thread.join()
def initialize(setupObjectToUse=None): if setupObjectToUse is None: configObjectToUse = Setup.config() else: configObjectToUse = setupObjectToUse.config with _mainRuntimeLock: if _mainRuntime[0] is not None: return #configure the main runtime compiler cfg = FORANative.RuntimeConfig() try: cfg.traceDefinitions = False cfg.traceArguments = False cfg.tracePaths = False cfg.useInlineMemoryManagement = True cfg.duplicateNativeEntrypoints = False cfg.validateVariablesDefinedBeforeUseInFlatCode = True cfg.inlineComplexity = 50 cfg.dynamicInlinerSleepTimeMilliseconds = 50 cfg.mediumPriorityCodeComplexityThreshold = 50 cfg.useLLVMOptimization = True cfg.unrollHotLoopsWithComparisons = True cfg.enableDoubleVectorStashing = True cfg.kickIntoInterpreterOnInline = True cfg.useReasoningCompiler = configObjectToUse.useReasoningCompiler cfg.disableSplitting = configObjectToUse.compilerDisableSplitting cfg.enableCodeExpansionRewriteRules = True cfg.applyRefcountOptimization = True cfg.extraDebugChecksDuringCompilation = False cfg.generateMachineCodeVectorAxioms = True cfg.sharedObjectLibraryPath = ufora.native.__file__ cfg.compilerThreadCount = configObjectToUse.foraCompilerThreads cfg.compilerDefinitionDumpDir = configObjectToUse.compilerDefinitionDumpDir cfg.instructionDefinitionDumpDir = configObjectToUse.instructionDefinitionDumpDir cfg.dynamicInlineCallThreshold = 10000 cfg.dynamicInlineCallThresholdSecondary = 5000 cfg.maxDynamicInlineComplexity = 10000 cfg.useDynamicInlining = True cfg.ptxLibraryPath = os.path.join(_curDir, "../CUDA/PTX/lib.ptx") cfg.compilerDiskCacheDir = configObjectToUse.compilerDiskCacheDir if cfg.compilerDefinitionDumpDir != "": logging.info("dumping CFGs to %s", cfg.compilerDefinitionDumpDir) try: os.system("rm " + cfg.compilerDefinitionDumpDir + " -rf") except: pass try: os.makedirs(cfg.compilerDefinitionDumpDir) except: pass axioms = open(os.path.join(_curDir, "../Axioms", "axioms.fora"), "r").read() _mainRuntime[0] = FORANative.initializeRuntime( axioms, cfg, ) if configObjectToUse.wantsPythonGilLoopChecker: ManagedThread.ManagedThread( target=FORANative.checkPythonGilLoop, args=( configObjectToUse.pythonGilLoopInterrupts, )).start() except: import traceback traceback.print_exc() raise