def start(self):
assert not self.hasStarted
if self.actuallyRunOutOfProcess:
def onStdout(msg):
logging.info("OutOfProcessDownloader Out> %s", msg)
def onStderr(msg):
logging.info("OutOfProcessDownloader Err> %s", msg)
self.childSubprocess = SubprocessRunner.SubprocessRunner(
[sys.executable, __file__,
str(self.childSocket.fileno())], onStdout, onStderr)
self.childSubprocess.start()
self.hasStarted = True
self.backgroundThread = ManagedThread.ManagedThread(
target=self.watchChild_)
self.backgroundThread.start()
else:
self.hasStarted = True
self.backgroundThread = ManagedThread.ManagedThread(
target=self.executeChild_)
self.backgroundThread.start()
def __init__(self, callbackScheduler, vdm, viewFactory):
Stoppable.Stoppable.__init__(self)
self.lock_ = threading.Lock()
self.callbackScheduler = callbackScheduler
self.definitionToIdMap_ = {}
self.idToDefinitionMap_ = {}
self.vdm = vdm
self.onJsonViewOfSystemChanged = None
self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
viewFactory.createView(retrySeconds=10.0, numRetries=10),
callbackScheduler)
self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)
self.cumulusClientId = CumulusNative.CumulusClientId(
Hash.Hash.sha1(str(uuid.uuid4())))
logging.info("CumulusClient created with %s", self.cumulusClientId)
self.cumulusClient = CumulusNative.CumulusClient(
vdm, self.cumulusClientId, self.callbackScheduler)
self.finalResponses = Queue.Queue()
self.cumulusClientListener = self.cumulusClient.createListener()
self.cpuAssignmentDependencyGraph = CumulusNative.CpuAssignmentDependencyGraph(
self.callbackScheduler.getFactory().createScheduler(
self.callbackScheduler.getMetadata() + "_cpuAssignmentGraph",
1), self.vdm)
self.cpuAssignmentDependencyGraph.subscribeToCumulusClient(
self.cumulusClient)
self.pendingCallbacksByGuid = {}
self.cpuAssignmentDependencyGraphListener = \
self.cpuAssignmentDependencyGraph.createListener()
self.threads = []
self.threads.append(
ManagedThread.ManagedThread(target=self.processClientMessages_,
args=()))
self.threads.append(
ManagedThread.ManagedThread(
target=self.processDependencyGraphMessages_, args=()))
for t in self.threads:
t.start()
self.nextCpuUpdateTime = time.time()
self.cpuMessagesSinceLastUpdate = 0
self.lastSystemwideUpdateTime = time.time()
def onConnectionAvailable(self, channels, clientOrMachineAndGuid):
def performConnect():
clientOrMachine, guid = clientOrMachineAndGuid
logging.info("Connection is available to %s with guid %s",
clientOrMachine, guid)
with self.lock:
if self.shouldStop():
logging.info(
"Rejecting a connection because we are no longer active"
)
for channel in channels:
channel.disconnect()
return
if clientOrMachine.isMachine():
with self.lock:
if clientOrMachine.asMachine.machine in self.droppedMachineIds:
return
self.connectedMachines.add(
clientOrMachine.asMachine.machine)
self.cumulusWorker.addMachine(
clientOrMachine.asMachine.machine, channels,
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
else:
self.cumulusWorker.addCumulusClient(
clientOrMachine.asClient.client, channels,
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
ManagedThread.ManagedThread(target=performConnect, args=()).start()
def onReconnectedToSharedState(self):
thread = ManagedThread.ManagedThread(
target=self.updatePersistentCacheView, args=())
self.reconnectPersistentCacheIndexViewThreads.append(thread)
thread.start()
def __init__(self, inMemoryCluster):
self.inMemoryCluster = inMemoryCluster
self._onClusterResponse = None
self.desires = {}
self.callbackQueue = Queue.Queue()
self.callbackThread = ManagedThread.ManagedThread(target=self.callbackLoop)
def onWorkerAdd(self, ip, ports, machineIdAsString):
machineId = CumulusNative.MachineId(Hash.Hash.stringToHash(machineIdAsString))
if machineId <= self.machineId:
logging.info("Worker %s detected worker %s, and waiting for incoming connection",
self.machineId,
machineId)
#only connect one way. If the worker is larger than us, then we connect to it
return
guid = Hash.Hash.sha1(str(uuid.uuid4()))
logging.info(
"Worker %s detected worker %s and initiating connection with guid %s",
self.machineId,
machineId,
guid
)
with self.lock:
# Track that we are trying to connect to this machine
self.connectingMachines.add(machineId)
ManagedThread.ManagedThread(
target=self.onWorkerAdd2, args=(machineId, ip, ports, guid)
).start()
def setDisconnected(self):
logging.warn("SharedState is disconnected from CAM. Trying to reconnect.")
with self._lock:
self._lastDisconnectTime = time.time()
self._lastReconnectTime = None
self.isConnected = False
def sleepAndTriggerUpdate():
while (not self.shouldStop() and (
self._lastReconnectTime is None or
time.time() <= self._lastReconnectTime + TIME_TO_SLEEP_AFTER_RECONNECT + 0.01)
):
time.sleep(0.01)
self.updateActiveMachinesOnReactorThread()
newThread = ManagedThread.ManagedThread(target = sleepAndTriggerUpdate)
with self._lock:
self._triggerUpdateAfterDisconnectThreads.append(newThread)
newThread.start()
if self.isConnecting:
logging.warn("CAM disconnected from SharedState while connecting. Setting a flag and sleeping.")
self.disconnectedWhileConnecting = True
connectDirectly = False
else:
connectDirectly = True
if connectDirectly:
self.connectView()
def __init__(self, vdm, offlineCache):
Stoppable.Stoppable.__init__(self)
self.dependencies_ = TwoWaySetMap.TwoWaySetMap()
self.vdm_ = vdm
self.offlineCache_ = offlineCache
self.finishedValuesAndTimeElapsed_ = {}
self.intermediates_ = {}
self.lock_ = threading.RLock()
self.completable_ = Queue.Queue()
self.timesComputed = 0
self.computingContexts_ = {}
self.computingContexts_t0_ = {}
self.isSplit_ = set()
self.watchers_ = {}
self.contexts_ = []
self.inProcessDownloader = (
OutOfProcessDownloader.OutOfProcessDownloaderPool(
Setup.config().cumulusServiceThreadCount,
actuallyRunOutOfProcess = False
)
)
self.threads_ = []
self.isActive = True
#setup the primary cache object, and set its worker threads going
for threadIx in range(Setup.config().cumulusServiceThreadCount):
workerThread = ManagedThread.ManagedThread(target = self.threadWorker)
workerThread.start()
self.threads_.append(workerThread)
def test_filtered_channel(self):
"""Verify that message filtering works
The sender sends a message that is expected to be filtered by FilteredChannel.
The test verifies that the receiver doesn't get the message
"""
channelFilter = FilteredChannelFactory.FilteredChannel(
callbackScheduler, self.filterMinId)
try:
sender, receiver = channelFilter.getChannelPair()
filteredMessage = SharedState.MessageOut.MinimumIdResponse(0)
sender.write(filteredMessage)
toGet = [None]
def tryGet():
try:
toGet[0] = receiver.get()
except UserWarning:
if channelFilter.stopFlag.is_set():
return
else:
raise
t = ManagedThread.ManagedThread(target=tryGet)
t.start()
t.join(1)
self.assertIsNone(toGet[0])
finally:
channelFilter.teardown()
def __init__(self, viewFactory):
Stoppable.Stoppable.__init__(self)
self.viewFactory = viewFactory
self.activeMachineIds = set()
self._outgoingNotifications = Queue.Queue()
self._lock = threading.RLock()
self._lastDisconnectTime = None
self._lastReconnectTime = None
self._registeredIpPortAndMachineId = None
logging.debug("Connecting to shared state: %s", viewFactory)
self.listeners_ = set()
self.clientID = None
self.isConnected = False
self.isConnecting = False
self.disconnectedWhileConnecting = False
self._triggerUpdateAfterDisconnectThreads = []
self.clientIdToIpPortAndMachineIdAsString = {}
self.machineIdToClientId = {}
self.workerStatusKeyspace = SharedState.Keyspace(
"ComparisonKeyType", NativeJson.Json(
(('P', 'CumulusNodeStatus'), )), 1)
self.asyncView = None
self.eventLoopThread = ManagedThread.ManagedThread(
target=self.eventDispatchLoop)
def test_shared_state_hang_on_view_id(self):
try:
def createFilteredChannelFactory(callbackScheduler, manager):
return FilteredChannelFactory.FilteredChannelFactory(
callbackScheduler, manager, self.filterInitialize)
harness = SharedStateTestHarness.SharedStateTestHarness(
True, inMemChannelFactoryFactory=createFilteredChannelFactory)
view = harness.viewFactory.createView()
def getViewId():
try:
view.id
except UserWarning:
pass
thread = ManagedThread.ManagedThread(target=getViewId)
thread.start()
thread.join(1)
self.assertTrue(thread.isAlive())
view.teardown()
thread.join()
finally:
harness.teardown()
def _channelConnectCallback(self, channel):
channel = channel.makeQueuelike(callbackScheduler)
t = ManagedThread.ManagedThread(target=self._echoLoop,
args=(channel, ))
t.start()
self.threads.append(t)
self.channels.append(t)
def simultaneously(*factories):
threads = [ManagedThread.ManagedThread(target=x) for x in factories]
for t in threads:
t.start()
for t in threads:
t.join()
def startLoop(logfileLocation, interval = 1.0):
global started
if started:
return
started = True
t = ManagedThread.ManagedThread(target = memoryWriteLoop, args = (logfileLocation,interval))
t.daemon = True
t.start()
def stressMultipleSharedStateReadWrites(self,
useTcpFactory=False,
keysToWrite=20,
threadcount=10):
keyspaceSize = keysToWrite * 5
subPasses = 10
if useTcpFactory:
viewFactory = ViewFactory.ViewFactory.TcpViewFactory(
callbackScheduler, address="localhost")
else:
viewFactory = self.simulator.getViewFactory()
worked = {}
for ix in range(threadcount):
worked[ix] = False
def test(threadIx):
for subPassIx in range(subPasses):
logging.info("Thread %s starting pass %s", threadIx, subPassIx)
testKeyspace = SharedState.Keyspace(
"TakeHighestIdKeyType", NativeJson.Json("TestSpace"), 1)
view = viewFactory.createView()
rng = SharedState.KeyRange(testKeyspace, 0, None, None, True,
False)
view.subscribe(rng)
for ix in range(keysToWrite):
with SharedState.Transaction(view):
ix = random.randint(0, keyspaceSize)
key = SharedState.Key(
testKeyspace, (NativeJson.Json("key %s" % ix), ))
value = uuid.uuid4().hex
view[key] = NativeJson.Json(value)
worked[threadIx] = True
threads = [
ManagedThread.ManagedThread(target=test, args=(ix, ))
for ix in range(threadcount)
]
for t in threads:
t.start()
for t in threads:
t.join()
expectedDict = {}
for ix in range(threadcount):
expectedDict[ix] = True
self.assertEqual(expectedDict, worked)
def __init__(self, numKeys, keySize):
self.q = Queue.Queue()
self.port = Setup.config().testPort
self.server = SimpleServer.SimpleServer(self.port)
self.server._onConnect = lambda sock, address : self.q.put(sock)
self.serverThread = ManagedThread.ManagedThread(target=self.server.start)
self.serverThread.daemon = True
self.serverThread.start()
ChannelTester.__init__(self, self.createSocketChannels, numKeys, keySize)
def startService(self):
with self.lock_:
if self.threads_:
return
logging.debug("Starting %s PythonIoTasks service threads",
Setup.config().externalDatasetLoaderServiceThreads)
for ix in range(self.threadcount):
self.threads_.append(
ManagedThread.ManagedThread(target=self.loadLoop))
self.threads_[-1].start()
def __init__(self, callbackScheduler, filterFun):
self.filterFun = filterFun
self.callbackScheduler = callbackScheduler
self.viewFacingViewChannel, self.filterFacingManagerChannel = SharedStateNative.InMemoryChannel(
self.callbackScheduler)
self.filterFacingViewChannel, self.managerFacingManagerChannel = SharedStateNative.InMemoryChannel(
self.callbackScheduler)
self.stopFlag = threading.Event()
self.channelPumpThreads = []
self.channelPumpThreads.append(
ManagedThread.ManagedThread(target=self.filteredChannelPump,
args=(self.filterFacingManagerChannel,
self.filterFacingViewChannel)))
self.channelPumpThreads.append(
ManagedThread.ManagedThread(
target=self.filteredChannelPump,
args=(self.filterFacingViewChannel,
self.filterFacingManagerChannel)))
for thread in self.channelPumpThreads:
thread.start()
def onChannelConnect(self, portIndex, cumulusChannel):
logging.debug("Received incoming connection on port id %d", portIndex)
with self.lock:
if self.shouldStop():
logging.info("Rejecting a connection because we are no longer active")
self._channelListener.rejectIncomingChannel(cumulusChannel)
return
# We process the incoming channel in a separate thread to minimize the
# amount of processing done in the socket-accept thread.
ManagedThread.ManagedThread(
target=self.doChannelHandshake, args=(cumulusChannel,)
).start()
def setUp(self):
logging.info("TestSocketChannel setting up")
self.port = Setup.config().testPort
self.serverChannel = None
self.server = None
self.serverChannelEvent = threading.Event()
self.tries = 0
self.server = TestServer(self.port, self.onConnect)
self.serverThread = ManagedThread.ManagedThread(
target=self.server.start)
self.serverThread.start()
self.server.blockUntilListening()
logging.info("Server thread is listening")
def __init__(self, callbackScheduler, cachePathOverride=None, port=None):
self.callbackScheduler = callbackScheduler
port = Setup.config().sharedStatePort
logging.info("Initializing SharedStateService with port = %s", port)
self.cachePath = cachePathOverride if cachePathOverride is not None else \
Setup.config().sharedStateCache
if self.cachePath != '' and not os.path.exists(self.cachePath):
os.makedirs(self.cachePath)
CloudService.Service.__init__(self)
self.socketServer = SimpleServer.SimpleServer(port)
self.keyspaceManager = KeyspaceManager(
0, 1, pingInterval=120, cachePathOverride=cachePathOverride)
self.socketServer._onConnect = self.onConnect
self.socketServerThread = ManagedThread.ManagedThread(
target=self.socketServer.start)
self.logfilePruneThread = ManagedThread.ManagedThread(
target=self.logFilePruner)
self.stoppedFlag = threading.Event()
def __init__(self, port, portScanIncrement=0):
assert isinstance(port, int)
assert isinstance(portScanIncrement, int)
self.requestedPort = port
self.port = port
self.portScanIncrement = portScanIncrement
self.socketServer = SimpleServer.SimpleServer(self.port)
self.boundEvent_ = threading.Event()
self.tornDown = False
self.onPortBound = None
self.socketConnectCallback = None
self.socketServer._onConnect = self.onSocketConnect
self.listenerThread = ManagedThread.ManagedThread(
target=self.socketServer.start)
def __init__(self, callbackScheduler, channelListener, sharedStateAddress):
Stoppable.Stoppable.__init__(self)
self._lock = threading.Lock()
self.callbackScheduler = callbackScheduler
self.sharedStateAddress = sharedStateAddress
self.channelListener = channelListener
self.channelListener.registerConnectCallback(
self.onSubscribableConnection)
ModuleImporter.initialize()
self.socketsToDisconnectOnExit = []
self.procsToKillOnExit = set()
self.isTornDown_ = False
self.cleanupThread = ManagedThread.ManagedThread(
target=self.cleanupThread_)
def test_socket_channel_shutdown(self):
done = threading.Event()
listener = ChannelListener.SocketListener(self.port)
listener.registerConnectCallback(lambda sock, address: done.set())
try:
thread = ManagedThread.ManagedThread(target=listener.start)
thread.start()
listener.blockUntilReady()
TcpChannelFactory.TcpStringChannelFactory(
callbackScheduler).createChannel(('localhost', self.port))
self.assertTrue(done.wait(2))
finally:
listener.stop()
thread.join()
def onWorkerAdd(self, ip, ports, machineIdAsString):
machineId = CumulusNative.MachineId(
HashNative.Hash.stringToHash(machineIdAsString))
with self.lock_:
if self.isTornDown_:
return
logging.info("CumulusClient %s preparing to connect to %s",
self.cumulusClientId, machineId)
self.desiredMachines_.add(machineId)
newThread = ManagedThread.ManagedThread(
target=self.addDesiredMachine, args=(machineId, ip, ports))
self.connectingThreads_.append(newThread)
self.connectingThreads_ = [
x for x in self.connectingThreads_ if x.isAlive()
]
newThread.start()
def __init__(self,
viewFactory,
onConnectCallback=lambda result: None,
onErrorCallback=lambda error: None):
self._stopFlag = threading.Event()
self._onConnectCallback = onConnectCallback
self._onErrorCallback = onErrorCallback
self._view = viewFactory.createView(retrySeconds=10.0, numRetries=10)
self._listener = SharedState.Listener(self._view)
self._subscribedKeyspaces = set()
self._keyspaceCallbackFunctions = {}
self._subscriptionDeferreds = {}
self._reactorThreadCalls = Queue.Queue()
self._viewReactorThread = ManagedThread.ManagedThread(
target=self._viewReactorLoop)
def _startChannelLoop(self, channelId, channel):
def channelLoop():
while not self.shouldStop:
try:
msg = channel.getTimeout(.1)
except UserWarning:
#we've been disconnected. propagate the disconnect and exit
self.onMessageReceived_("", channelId)
return
except:
self.onMessageReceived_("", channelId)
raise
if msg is not None:
self.onMessageReceived_(msg, channelId)
channel.disconnect()
self.channelThreads[channelId] = ManagedThread.ManagedThread(
target=channelLoop)
self.channelThreads[channelId].start()
def test_filtered_channels_noblock(self):
channelFilter = FilteredChannelFactory.FilteredChannel(
callbackScheduler, self.filterAllButMinId)
try:
viewChannel, managerChannel = channelFilter.getChannelPair()
MessageOut = getattr(SharedStateNative, 'SharedState::MessageOut')
filteredMessage = MessageOut.MinimumIdResponse(0)
allowedMessage = MessageOut.FlushRequest(0)
viewChannel.write(filteredMessage)
viewChannel.write(allowedMessage)
toGet = [None]
def tryGet():
toGet[0] = managerChannel.get()
t = ManagedThread.ManagedThread(target=tryGet)
t.start()
t.join(4)
self.assertIsNotNone(toGet[0])
finally:
channelFilter.teardown()
def test_socket_listener(self):
server = ChannelEchoServer(self.port)
try:
thread = ManagedThread.ManagedThread(target=server.start)
thread.start()
server.blockUntilReady()
stringChannelFactory = TcpChannelFactory.TcpStringChannelFactory(
callbackScheduler)
channel = stringChannelFactory.createChannel(
('localhost', self.port))
channel = channel.makeQueuelike(callbackScheduler)
toSend = "Hi There!"
channel.write(toSend)
self.assertEquals(toSend, channel.get())
finally:
try:
server.teardown()
except UserWarning:
pass
thread.join()
def initialize(setupObjectToUse=None):
if setupObjectToUse is None:
configObjectToUse = Setup.config()
else:
configObjectToUse = setupObjectToUse.config
with _mainRuntimeLock:
if _mainRuntime[0] is not None:
return
#configure the main runtime compiler
cfg = FORANative.RuntimeConfig()
try:
cfg.traceDefinitions = False
cfg.traceArguments = False
cfg.tracePaths = False
cfg.useInlineMemoryManagement = True
cfg.duplicateNativeEntrypoints = False
cfg.validateVariablesDefinedBeforeUseInFlatCode = True
cfg.inlineComplexity = 50
cfg.dynamicInlinerSleepTimeMilliseconds = 50
cfg.mediumPriorityCodeComplexityThreshold = 50
cfg.useLLVMOptimization = True
cfg.unrollHotLoopsWithComparisons = True
cfg.enableDoubleVectorStashing = True
cfg.kickIntoInterpreterOnInline = True
cfg.useReasoningCompiler = configObjectToUse.useReasoningCompiler
cfg.disableSplitting = configObjectToUse.compilerDisableSplitting
cfg.enableCodeExpansionRewriteRules = True
cfg.applyRefcountOptimization = True
cfg.extraDebugChecksDuringCompilation = False
cfg.generateMachineCodeVectorAxioms = True
cfg.sharedObjectLibraryPath = ufora.native.__file__
cfg.compilerThreadCount = configObjectToUse.foraCompilerThreads
cfg.compilerDefinitionDumpDir = configObjectToUse.compilerDefinitionDumpDir
cfg.instructionDefinitionDumpDir = configObjectToUse.instructionDefinitionDumpDir
cfg.dynamicInlineCallThreshold = 10000
cfg.dynamicInlineCallThresholdSecondary = 5000
cfg.maxDynamicInlineComplexity = 10000
cfg.useDynamicInlining = True
cfg.ptxLibraryPath = os.path.join(_curDir, "../CUDA/PTX/lib.ptx")
cfg.compilerDiskCacheDir = configObjectToUse.compilerDiskCacheDir
if cfg.compilerDefinitionDumpDir != "":
logging.info("dumping CFGs to %s",
cfg.compilerDefinitionDumpDir)
try:
os.system("rm " + cfg.compilerDefinitionDumpDir + " -rf")
except:
pass
try:
os.makedirs(cfg.compilerDefinitionDumpDir)
except:
pass
axioms = open(os.path.join(_curDir, "../Axioms", "axioms.fora"),
"r").read()
_mainRuntime[0] = FORANative.initializeRuntime(
axioms,
cfg,
)
if configObjectToUse.wantsPythonGilLoopChecker:
ManagedThread.ManagedThread(
target=FORANative.checkPythonGilLoop,
args=(
configObjectToUse.pythonGilLoopInterrupts, )).start()
except:
import traceback
traceback.print_exc()
raise
