def performConnect():
clientOrMachine, guid = clientOrMachineAndGuid
logging.info("Connection is available to %s with guid %s", clientOrMachine, guid)
with self.lock:
if self.shouldStop():
logging.info("Rejecting a connection because we are no longer active")
for channel in channels:
channel.disconnect()
return
if clientOrMachine.isMachine():
with self.lock:
if clientOrMachine.asMachine.machine in self.droppedMachineIds:
return
self.connectedMachines.add(clientOrMachine.asMachine.machine)
self.cumulusWorker.addMachine(
clientOrMachine.asMachine.machine,
channels,
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler
)
else:
self.cumulusWorker.addCumulusClient(
clientOrMachine.asClient.client,
channels,
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler
)
def performConnect():
clientOrMachine, guid = clientOrMachineAndGuid
logging.info("Connection is available to %s with guid %s", clientOrMachine, guid)
with self.lock:
if self.shouldStop():
logging.info("Rejecting a connection because we are no longer active")
for channel in channels:
channel.disconnect()
return
if clientOrMachine.isMachine():
with self.lock:
if clientOrMachine.asMachine.machine in self.droppedMachineIds:
return
self.connectedMachines.add(clientOrMachine.asMachine.machine)
self.cumulusWorker.addMachine(
clientOrMachine.asMachine.machine,
channels,
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler
)
else:
self.cumulusWorker.addCumulusClient(
clientOrMachine.asClient.client,
channels,
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler
)
def __init__(self,
callbackScheduler,
vdm,
channelFactory,
activeMachines,
viewFactory):
CumulusGateway.CumulusGateway.__init__(self,
callbackScheduler,
vdm,
viewFactory)
ModuleImporter.initialize()
self.channelFactory_ = channelFactory
self.connectedMachines_ = set()
self.disconnectedMachines_ = set()
self.desiredMachines_ = set()
self.connectingThreads_ = []
self.isTornDown_ = False
self.activeMachines = activeMachines
self.activeMachines.addListener(self)
self.activeMachines.startService()
def connectToRemoteWorker(self, machineId, ip, port, guid):
logging.info("Attempting to connect to machine %s on %s:%s with guid %s",
machineId,
ip,
port,
guid)
stringChannel = self._channelFactory.createChannel((ip, port))
stringChannel.write(ufora.version)
stringChannel.write(machineId.__getstate__())
#initiate the handshake
stringChannel.write(
CumulusNative.CumulusClientOrMachine.Machine(
self.machineId
).__getstate__()
)
stringChannel.write(guid.__getstate__())
logging.info("CumulusService %s wrote handshake for %s with guid %s",
self.machineId,
machineId,
guid
)
channelAsQueue = stringChannel.makeQueuelike(self.callbackScheduler)
msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)
if msg is None:
logging.error("While attempting to add worker %s with guid %s, " +
"CumulusWorker %s did not receive a builtin hash message during handshake",
machineId,
guid,
self.machineId
)
return None
otherWorkersBuiltinHash = Hash.Hash(0)
otherWorkersBuiltinHash.__setstate__(msg)
builtinsAgree = otherWorkersBuiltinHash == ModuleImporter.builtinModuleImplVal().hash
if not builtinsAgree:
logging.critical("CumulusWorker %s could not connect to CumulusWorker %s as they have " + \
"different builtins; former's builtin hash: %s, latter's builtin hash: " + \
"%s",
self.machineId,
machineId,
ModuleImporter.builtinModuleImplVal().hash,
otherWorkersBuiltinHash
)
channelAsQueue.disconnect()
return None
return channelAsQueue
def initialize(setupObjectToUse = None, useLocalEvaluator = True):
global _builtin
if _builtin is not None:
return
Runtime.initialize(setupObjectToUse)
ModuleImporter.initialize(setupObjectToUse)
Evaluator.initialize(setupObjectToUse, useLocalEvaluator)
_builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
def importModule(modulePath):
#TODO BUG anybody: why is this here? It was getting passed as the
#searchForFreeVariables argument to importModuleFromPath for some reason
ModuleImporter.builtinModuleImplVal()
return ForaValue.FORAValue(
ModuleImporter.importModuleFromPath(
modulePath,
True
)
)
def initialize(setupObjectToUse=None, useLocalEvaluator=True):
global _builtin
if _builtin is not None:
return
Runtime.initialize(setupObjectToUse)
ModuleImporter.initialize(setupObjectToUse)
Evaluator.initialize(setupObjectToUse, useLocalEvaluator)
_builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
def connectToRemoteWorker(self, machineId, ip, port, guid):
logging.info(
"Attempting to connect to machine %s on %s:%s with guid %s",
machineId, ip, port, guid)
stringChannel = self._channelFactory.createChannel((ip, port))
stringChannel.write(ufora.version)
stringChannel.write(machineId.__getstate__())
#initiate the handshake
stringChannel.write(
CumulusNative.CumulusClientOrMachine.Machine(
self.machineId).__getstate__())
stringChannel.write(guid.__getstate__())
logging.info("CumulusService %s wrote handshake for %s with guid %s",
self.machineId, machineId, guid)
channelAsQueue = stringChannel.makeQueuelike(self.callbackScheduler)
msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)
if msg is None:
logging.error(
"While attempting to add worker %s with guid %s, " +
"CumulusWorker %s did not receive a builtin hash message during handshake",
machineId, guid, self.machineId)
return None
otherWorkersBuiltinHash = Hash.Hash(0)
otherWorkersBuiltinHash.__setstate__(msg)
builtinsAgree = otherWorkersBuiltinHash == ModuleImporter.builtinModuleImplVal(
).hash
if not builtinsAgree:
logging.critical("CumulusWorker %s could not connect to CumulusWorker %s as they have " + \
"different builtins; former's builtin hash: %s, latter's builtin hash: " + \
"%s",
self.machineId,
machineId,
ModuleImporter.builtinModuleImplVal().hash,
otherWorkersBuiltinHash
)
channelAsQueue.disconnect()
return None
return channelAsQueue
def __init__(self, callbackScheduler, channelListener, sharedStateAddress):
Stoppable.Stoppable.__init__(self)
self._lock = threading.Lock()
self.callbackScheduler = callbackScheduler
self.sharedStateAddress = sharedStateAddress
self.channelListener = channelListener
self.channelListener.registerConnectCallback(self.onSubscribableConnection)
ModuleImporter.initialize()
self.socketsToDisconnectOnExit = []
self.procsToKillOnExit = set()
self.isTornDown_ = False
self.cleanupThread = ManagedThread.ManagedThread(target=self.cleanupThread_)
def connectToWorker(self, machineId, ip, port, guid):
with self.lock_:
stringChannel = self.channelFactory_.createChannel((ip, port))
builtinsHash = ModuleImporter.builtinModuleImplVal().hash
clientId = self.cumulusClientId
callbackScheduler = self.callbackScheduler
logging.info("Client %s writing version message '%s' to %s", clientId, ufora.version, machineId)
stringChannel.write(ufora.version)
logging.info("Client %s writing client ID message to %s", clientId, machineId)
stringChannel.write(machineId.__getstate__())
logging.info("Client %s writing expected machineId message to %s", clientId, machineId)
stringChannel.write(CumulusNative.CumulusClientOrMachine.Client(clientId).__getstate__())
logging.info("Client %s writing guid %s to %s", clientId, guid, machineId)
stringChannel.write(guid.__getstate__())
channelAsQueue = stringChannel.makeQueuelike(callbackScheduler)
msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)
if msg is None:
logging.error(
"While attempting to add worker %s, CumulusClient %s did not "
+ "receive a builtin hash message during handshake",
machineId,
clientId,
)
assert msg is not None
logging.info("Client %s received serialized worker's builtin hash", clientId)
try:
workersBuiltinHash = HashNative.Hash(0)
workersBuiltinHash.__setstate__(msg)
except:
logging.info("Client received a bad worker hash: %s of size %s", repr(msg), len(msg))
raise
builtinsAgree = workersBuiltinHash == builtinsHash
if not builtinsAgree:
logging.critical(
"Could not connect CumulusClient %s to CumulusWorker %s as they "
+ "have different builtins; client's builtin hash: %s, worker's "
+ "builtin hash: %s. Disconnecting channel",
clientId,
machineId,
builtinsHash,
workersBuiltinHash,
)
channelAsQueue.disconnect()
return None
return channelAsQueue
def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm, **kwds):
s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory()
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
1,
0,
s3Service=s3Service,
memoryPerWorkerMB=400,
callbackScheduler=callbackScheduler,
threadsPerWorker=Setup.config().cumulusServiceThreadCount,
**kwds
)
CumulusGateway.CumulusGateway.__init__(self,
callbackScheduler,
vdm,
simulation.sharedStateViewFactory)
self.s3Service = s3Service
self.simulation = simulation
self.viewFactory = self.simulation.sharedStateViewFactory
self.worker = self.simulation.getWorker(0)
self.workerVdm = self.simulation.getWorkerVdm(0)
channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel(self.callbackScheduler)
machineId = self.workerVdm.getMachineId()
self.cumulusClient.addMachine(
machineId,
[channel1Client, channel2Client],
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler
)
self.worker.addCumulusClient(
self.cumulusClientId,
[channel1Worker, channel2Worker],
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler
)
self.loadingService = self.simulation.loadingServices[0]
def constructConverter(purePythonModuleImplVal, vdm):
pythonNameToPyforaName = {}
pythonNameToPyforaName.update(NamedSingletons.pythonNameToPyforaName)
pythonNameToPyforaName.update(PyAbortSingletons.pythonNameToPyforaName)
return ForaNative.PythonBinaryStreamToImplval(
vdm, purePythonModuleImplVal, ModuleImporter.builtinModuleImplVal(),
pythonNameToPyforaName, PythonAstConverter.parseStringToPythonAst)
def __init__(self, callbackScheduler, channelListener, sharedStateAddress):
Stoppable.Stoppable.__init__(self)
self._lock = threading.Lock()
self.callbackScheduler = callbackScheduler
self.sharedStateAddress = sharedStateAddress
self.channelListener = channelListener
self.channelListener.registerConnectCallback(
self.onSubscribableConnection)
ModuleImporter.initialize()
self.socketsToDisconnectOnExit = []
self.procsToKillOnExit = set()
self.isTornDown_ = False
self.cleanupThread = ManagedThread.ManagedThread(
target=self.cleanupThread_)
def onWorkerAdd2(self, machineId, ip, ports, guid):
logging.debug(
"CumulusService %s adding worker %s and connecting to it with guid %s",
self.machineId,
machineId,
guid
)
assert len(ports) == 2
channels = []
while self.shouldConnectToMachine(machineId):
try:
for channel in channels:
channel.disconnect()
channels = []
for port in ports:
channel = None
tries = 0
while channel is None and tries < 3:
channel = self.connectToRemoteWorker(machineId, ip, port, guid)
tries += 1
if channel is None:
logging.error(
"Unable to connect to worker %s on %s:%s with guid %s.",
machineId,
ip,
port,
guid
)
continue
channels.append(channel)
with self.lock:
if machineId in self.droppedMachineIds:
logging.warn("not accepting %s", machineId)
self.connectedMachines.add(machineId)
self.connectingMachines.discard(machineId)
self.cumulusWorker.addMachine(
machineId,
channels,
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler
)
logging.info("Connection is available to Worker %s with guid %s",
machineId,
guid)
return
except:
logging.error("Failed to add worker %s: %s", machineId, traceback.format_exc())
def onWorkerAdd2(self, machineId, ip, ports, guid):
logging.debug(
"CumulusService %s adding worker %s and connecting to it with guid %s",
self.machineId,
machineId,
guid
)
assert len(ports) == 2
channels = []
while self.shouldConnectToMachine(machineId):
try:
for channel in channels:
channel.disconnect()
channels = []
for port in ports:
channel = None
tries = 0
while channel is None and tries < 3:
channel = self.connectToRemoteWorker(machineId, ip, port, guid)
tries += 1
if channel is None:
logging.error(
"Unable to connect to worker %s on %s:%s with guid %s.",
machineId,
ip,
port,
guid
)
continue
channels.append(channel)
with self.lock:
if machineId in self.droppedMachineIds:
logging.warn("not accepting %s", machineId)
self.connectedMachines.add(machineId)
self.connectingMachines.discard(machineId)
self.cumulusWorker.addMachine(
machineId,
channels,
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler
)
logging.info("Connection is available to Worker %s with guid %s",
machineId,
guid)
return
except:
logging.error("Failed to add worker %s: %s", machineId, traceback.format_exc())
def __init__(self, callbackScheduler, vdm, channelFactory, activeMachines,
viewFactory):
CumulusGateway.CumulusGateway.__init__(self, callbackScheduler, vdm,
viewFactory)
ModuleImporter.initialize()
self.channelFactory_ = channelFactory
self.connectedMachines_ = set()
self.disconnectedMachines_ = set()
self.desiredMachines_ = set()
self.connectingThreads_ = []
self.isTornDown_ = False
self.activeMachines = activeMachines
self.activeMachines.addListener(self)
self.activeMachines.startService()
def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm,
**kwds):
s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory()
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
1,
0,
s3Service=s3Service,
memoryPerWorkerMB=400,
callbackScheduler=callbackScheduler,
threadsPerWorker=Setup.config().cumulusServiceThreadCount,
**kwds)
CumulusGateway.CumulusGateway.__init__(
self, callbackScheduler, vdm, simulation.sharedStateViewFactory)
self.s3Service = s3Service
self.simulation = simulation
self.viewFactory = self.simulation.sharedStateViewFactory
self.worker = self.simulation.getWorker(0)
self.workerVdm = self.simulation.getWorkerVdm(0)
channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
machineId = self.workerVdm.getMachineId()
self.cumulusClient.addMachine(machineId,
[channel1Client, channel2Client],
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
self.worker.addCumulusClient(self.cumulusClientId,
[channel1Worker, channel2Worker],
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
self.loadingService = self.simulation.loadingServices[0]
def canonicalPurePythonModule():
if canonicalPurePythonModuleCache_[0] is None:
path = os.path.join(os.path.abspath(os.path.split(pyfora.__file__)[0]), "fora")
moduleTree = ModuleDirectoryStructure.ModuleDirectoryStructure.read(path, "purePython", "fora")
canonicalPurePythonModuleCache_[0] = ModuleImporter.importModuleFromMDS(
moduleTree,
"fora",
"purePython",
searchForFreeVariables=True
)
return canonicalPurePythonModuleCache_[0]
def constructConverter(purePythonModuleImplval, vdm):
if purePythonModuleImplval is None:
return Converter(vdmOverride=vdm)
else:
singletonAndExceptionConverter = \
PyforaSingletonAndExceptionConverter.PyforaSingletonAndExceptionConverter(
purePythonModuleImplval
)
primitiveTypeMapping = {
bool: purePythonModuleImplval.getObjectMember("PyBool"),
str: purePythonModuleImplval.getObjectMember("PyString"),
int: purePythonModuleImplval.getObjectMember("PyInt"),
float: purePythonModuleImplval.getObjectMember("PyFloat"),
type(None): purePythonModuleImplval.getObjectMember("PyNone"),
}
nativeConstantConverter = ForaNative.PythonConstantConverter(
primitiveTypeMapping
)
nativeListConverter = ForaNative.makePythonListConverter(
purePythonModuleImplval.getObjectMember("PyList")
)
nativeTupleConverter = ForaNative.makePythonTupleConverter(
purePythonModuleImplval.getObjectMember("PyTuple")
)
nativeDictConverter = ForaNative.makePythonDictConverter(
purePythonModuleImplval.getObjectMember("PyDict")
)
foraBuiltinsImplVal = ModuleImporter.builtinModuleImplVal()
return Converter(
nativeListConverter=nativeListConverter,
nativeTupleConverter=nativeTupleConverter,
nativeDictConverter=nativeDictConverter,
nativeConstantConverter=nativeConstantConverter,
singletonAndExceptionConverter=singletonAndExceptionConverter,
vdmOverride=vdm,
purePythonModuleImplVal=purePythonModuleImplval,
foraBuiltinsImplVal=foraBuiltinsImplVal
)
def tryOnWorkerAdd(self, machineId, ip, ports):
if self.isTornDown_:
return False
guid = HashNative.Hash.sha1(str(uuid.uuid4()))
try:
# TODO: get the number of cumulus ports from config
assert len(ports) == 2
channels = []
for i in range(2):
channel = self.connectToWorker(machineId, ip, ports[i], guid)
assert channel is not None
channels.append(channel)
logging.info("CumulusClient %s successfully connected to both channels of %s",
self.cumulusClientId,
machineId
)
with self.lock_:
if machineId not in self.desiredMachines_:
return False
with self.lock_:
if machineId in self.disconnectedMachines_:
return True
self.cumulusClient.addMachine(
machineId,
channels,
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler
)
self.connectedMachines_.add(machineId)
self.desiredMachines_.discard(machineId)
return True
except:
logging.error("Failed: %s", traceback.format_exc())
return False
def tryOnWorkerAdd(self, machineId, ip, ports):
if self.isTornDown_:
return False
guid = HashNative.Hash.sha1(str(uuid.uuid4()))
try:
# TODO: get the number of cumulus ports from config
assert len(ports) == 2
channels = []
for i in range(2):
channel = self.connectToWorker(machineId, ip, ports[i], guid)
assert channel is not None
channels.append(channel)
logging.info(
"CumulusClient %s successfully connected to both channels of %s",
self.cumulusClientId, machineId)
with self.lock_:
if machineId not in self.desiredMachines_:
return False
with self.lock_:
if machineId in self.disconnectedMachines_:
return True
self.cumulusClient.addMachine(
machineId, channels, ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
self.connectedMachines_.add(machineId)
self.desiredMachines_.discard(machineId)
return True
except:
logging.error("Failed: %s", traceback.format_exc())
return False
def reloadBuiltin():
global _builtin
ModuleImporter.initialize(reimport = True)
_builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
def connectToRemoteWorker(self, machineId, ip, port, guid):
logging.debug("Attempting to connect to machine %s on %s:%s with guid %s",
machineId,
ip,
port,
guid)
try:
stringChannel = self._channelFactory.createChannel((ip, port))
except:
logging.error("CAN'T CONNECT TO WORKER ON %s:%s!\n"
"This may be a temporary failure but if the problem persists, "
"check the workers' network configuration and verify "
"that the machines can see each other.",
ip, port)
raise
stringChannel.write(ufora.version)
stringChannel.write(machineId.__getstate__())
#initiate the handshake
stringChannel.write(
CumulusNative.CumulusClientOrMachine.Machine(
self.machineId
).__getstate__()
)
stringChannel.write(guid.__getstate__())
logging.debug("CumulusService %s wrote handshake for %s with guid %s",
self.machineId,
machineId,
guid)
channelAsQueue = stringChannel.makeQueuelike(self.callbackScheduler)
msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)
if msg is None:
logging.error("CAN'T CONNECT TO WORKER ON %s:%s!\n"
"While attempting to add worker %s with guid %s, "
"Worker %s did not receive a builtin hash message "
"during handshake.\n"
"Verify that the ufora worker is running on the remote machine.",
ip, port,
machineId, guid,
self.machineId)
return None
otherWorkersBuiltinHash = Hash.Hash(0)
otherWorkersBuiltinHash.__setstate__(msg)
builtinsAgree = otherWorkersBuiltinHash == ModuleImporter.builtinModuleImplVal().hash
if not builtinsAgree:
logging.critical("CAN'T CONNECT TO WORKER ON %s:%s!\n"
"Worker %s could not connect to Worker %s as they have "
"different builtins; former's builtin hash: %s, latter's builtin hash: "
"%s\n"
"Verify that both machines run the same ufora version.",
ip, port,
self.machineId,
machineId,
ModuleImporter.builtinModuleImplVal().hash,
otherWorkersBuiltinHash)
channelAsQueue.disconnect()
return None
return channelAsQueue
def __init__(self, callbackScheduler, sharedStateViewFactory, computedValueGatewayFactory):
self.lock = threading.Lock()
self.cacheLoadEvents = {}
self.resultsById_ = {}
self.eventsById_ = {}
logging.info("created a component host")
self.graph = ComputedGraph.ComputedGraph()
logging.info("created a ComputedGraph")
Runtime.initialize()
logging.info("Runtime initialized")
ModuleImporter.initialize()
logging.info("Module importer initialized")
Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
self.incomingObjectCache = IncomingObjectCache()
self.outgoingObjectCache = OutgoingObjectCache()
self.VDM = VectorDataManager.constructVDM(callbackScheduler)
self.VDM.setDropUnreferencedPagesWhenFull(True)
logging.info("created a VDM")
logging.info("got shared state view factory: %s", sharedStateViewFactory)
def initValueGateway():
with self.graph:
self.computedValueGateway = computedValueGatewayFactory()
self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway
def initSynchronizer():
self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
logging.info("created a SharedStateSynchronizer")
self.synchronizer.attachView(sharedStateViewFactory.createView())
logging.info("attached shared state view.")
simultaneously(initSynchronizer, initValueGateway)
self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()
self.outstandingMessagesById = {}
self.expectedMessageId = 0
self.messageTypeHandlers = {}
self.messageTypeHandlers["Read"] = self.handleReadMessage
self.messageTypeHandlers["Assign"] = self.handleAssignMessage
self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds
self.pendingObjectQueue = []
self.subscriptions = Subscriptions.Subscriptions(self.graph, self.computedValueGateway, self.synchronizer)
def __init__(self,
ownAddress,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnosticsDir,
config,
viewFactory,
s3InterfaceFactory=None,
objectStore=None):
Stoppable.Stoppable.__init__(self)
#acquire a machineId randomly, using uuid
self.machineId = CumulusNative.MachineId(
Hash.Hash.sha1(str(uuid.uuid4()))
)
self.ownAddress = ownAddress
self.callbackScheduler = callbackScheduler
self.viewFactory = viewFactory
self.s3InterfaceFactory = s3InterfaceFactory
self.objectStore = objectStore
self.threadsStarted_ = False
self.connectedMachines = set()
self.connectingMachines = set() # machines we are in the process of connecting to
self.droppedMachineIds = set()
self.lock = threading.RLock()
self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
self.cumulusTrackTcmalloc = config.cumulusTrackTcmalloc
self.eventHandler = eventHandler
self.reconnectPersistentCacheIndexViewThreads = []
if config.cumulusDiskCacheStorageSubdirectory is not None:
self.cumulusDiskCacheWantsDeletionOnTeardown = True
self.cumulusDiskCacheStorageDir = os.path.join(
config.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageSubdirectory
)
else:
self.cumulusDiskCacheWantsDeletionOnTeardown = False
self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir
self._stopEvent = threading.Event()
self._channelListener = channelListener
assert len(self._channelListener.ports) == 2
self._channelFactory = channelFactory
Runtime.initialize()
ModuleImporter.initialize()
self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
self.viewFactory
)
self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
target=self._channelListener.start
)
self.vdm = VectorDataManager.constructVDM(
callbackScheduler,
self.cumulusVectorRamCacheSizeOverride,
self.cumulusMaxRamCacheSizeOverride
)
if self.cumulusTrackTcmalloc:
self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()
self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
viewFactory.createView(retrySeconds=10.0, numRetries=10),
callbackScheduler
)
self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)
self.deleteCumulusDiskCacheIfNecessary()
self.offlineCache = CumulusNative.DiskOfflineCache(
callbackScheduler,
self.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageMB * 1024 * 1024,
config.cumulusDiskCacheStorageFileCount
)
#If the "s3InterfaceFactory" is not in-memory, we use real out of process python.
#it would be better if this were more explicit
outOfProcess = self.s3InterfaceFactory is not None and self.s3InterfaceFactory.isCompatibleWithOutOfProcessDownloadPool
self.outOfProcessPythonTasks = OutOfProcessPythonTasks.OutOfProcessPythonTasks(outOfProcess=outOfProcess)
self.vdm.initializeOutOfProcessPythonTasks(self.outOfProcessPythonTasks.nativeTasks)
checkpointInterval = config.cumulusCheckpointIntervalSeconds
if checkpointInterval == 0:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
else:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
checkpointInterval,
1024 * 1024
)
self.cumulusWorker = self.constructCumlusWorker(
callbackScheduler,
CumulusNative.CumulusWorkerConfiguration(
self.machineId,
self.cumulusThreadCountOverride,
checkpointPolicy,
ExecutionContext.createContextConfiguration(),
diagnosticsDir or ""
),
self.vdm,
self.offlineCache,
eventHandler
)
self.datasetLoadService = None
if self.s3InterfaceFactory:
externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
callbackScheduler
)
self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
self.s3InterfaceFactory,
self.objectStore,
self.vdm,
externalDatasetChannel.makeQueuelike(callbackScheduler)
)
self.cumulusWorker.startComputations()
if self.datasetLoadService:
self.datasetLoadService.startService()
def __init__(self, callbackSchedulerFactory, callbackScheduler, vdm,
**kwds):
#don't modify callers directly
kwds = dict(kwds)
if 's3Service' in kwds:
s3Service = kwds['s3Service']
del kwds['s3Service']
else:
s3Service = InMemoryS3Interface.InMemoryS3InterfaceFactory()
if 'threadsPerWorker' in kwds:
threadsPerWorker = kwds['threadsPerWorker']
del kwds['threadsPerWorker']
else:
threadsPerWorker = Setup.config().cumulusServiceThreadCount
if 'memoryPerWorkerMB' in kwds:
memoryPerWorkerMB = kwds['memoryPerWorkerMB']
del kwds['memoryPerWorkerMB']
else:
memoryPerWorkerMB = 400
if 'maxMBPerOutOfProcessPythonTask' in kwds:
maxBytesPerOutOfProcessPythonTask = kwds[
'maxMBPerOutOfProcessPythonTask'] * 1024 * 1024
del kwds['maxMBPerOutOfProcessPythonTask']
else:
maxBytesPerOutOfProcessPythonTask = None
workerCount = 1
if 'workerCount' in kwds:
workerCount = kwds['workerCount']
del kwds['workerCount']
simulation = InMemoryCumulusSimulation.InMemoryCumulusSimulation(
workerCount,
0,
s3Service=s3Service,
memoryPerWorkerMB=memoryPerWorkerMB,
callbackScheduler=callbackScheduler,
threadsPerWorker=threadsPerWorker,
maxBytesPerOutOfProcessPythonTask=maxBytesPerOutOfProcessPythonTask,
**kwds)
CumulusGateway.CumulusGateway.__init__(
self, callbackScheduler, vdm, simulation.sharedStateViewFactory)
self.s3Service = s3Service
self.simulation = simulation
self.viewFactory = self.simulation.sharedStateViewFactory
for workerIx in xrange(workerCount):
worker = self.simulation.getWorker(workerIx)
workerVdm = self.simulation.getWorkerVdm(workerIx)
channel1Client, channel1Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
channel2Client, channel2Worker = StringChannelNative.InMemoryStringChannel(
self.callbackScheduler)
machineId = workerVdm.getMachineId()
self.cumulusClient.addMachine(
machineId, [channel1Client, channel2Client],
ModuleImporter.builtinModuleImplVal(), self.callbackScheduler)
worker.addCumulusClient(self.cumulusClientId,
[channel1Worker, channel2Worker],
ModuleImporter.builtinModuleImplVal(),
self.callbackScheduler)
self.worker = self.simulation.getWorker(0)
self.workerVdm = self.simulation.getWorkerVdm(0)
self.loadingService = self.simulation.loadingServices[0]
def reloadBuiltin():
global _builtin
ModuleImporter.initialize(reimport=True)
_builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
def connectToWorker(self, machineId, ip, port, guid):
with self.lock_:
stringChannel = self.channelFactory_.createChannel((ip, port))
builtinsHash = ModuleImporter.builtinModuleImplVal().hash
clientId = self.cumulusClientId
callbackScheduler = self.callbackScheduler
logging.info("Client %s writing version message '%s' to %s", clientId,
ufora.version, machineId)
stringChannel.write(ufora.version)
logging.info("Client %s writing client ID message to %s", clientId,
machineId)
stringChannel.write(machineId.__getstate__())
logging.info("Client %s writing expected machineId message to %s",
clientId, machineId)
stringChannel.write(
CumulusNative.CumulusClientOrMachine.Client(
clientId).__getstate__())
logging.info("Client %s writing guid %s to %s", clientId, guid,
machineId)
stringChannel.write(guid.__getstate__())
channelAsQueue = stringChannel.makeQueuelike(callbackScheduler)
msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)
if msg is None:
logging.error(
"While attempting to add worker %s, CumulusClient %s did not "
+ "receive a builtin hash message during handshake", machineId,
clientId)
assert msg is not None
logging.info("Client %s received serialized worker's builtin hash",
clientId)
try:
workersBuiltinHash = HashNative.Hash(0)
workersBuiltinHash.__setstate__(msg)
except:
logging.info("Client received a bad worker hash: %s of size %s",
repr(msg), len(msg))
raise
builtinsAgree = workersBuiltinHash == builtinsHash
if not builtinsAgree:
logging.critical("Could not connect CumulusClient %s to CumulusWorker %s as they " + \
"have different builtins; client's builtin hash: %s, worker's " + \
"builtin hash: %s. Disconnecting channel",
clientId,
machineId,
builtinsHash,
workersBuiltinHash
)
channelAsQueue.disconnect()
return None
return channelAsQueue
def __init__(self,
ownAddress,
channelListener,
channelFactory,
eventHandler,
callbackScheduler,
diagnosticsDir,
config,
viewFactory):
Stoppable.Stoppable.__init__(self)
#acquire a machineId randomly, using uuid
self.machineId = CumulusNative.MachineId(
Hash.Hash.sha1(str(uuid.uuid4()))
)
self.ownAddress = ownAddress
self.callbackScheduler = callbackScheduler
self.viewFactory = viewFactory
self.threadsStarted_ = False
self.connectedMachines = set()
self.connectingMachines = set() # machines we are in the process of connecting to
self.droppedMachineIds = set()
self.lock = threading.RLock()
self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024*1024
self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024*1024
self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc
self.reconnectPersistentCacheIndexViewThreads = []
if config.cumulusDiskCacheStorageSubdirectory is not None:
self.cumulusDiskCacheWantsDeletionOnTeardown = True
self.cumulusDiskCacheStorageDir = os.path.join(
config.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageSubdirectory
)
else:
self.cumulusDiskCacheWantsDeletionOnTeardown = False
self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir
logging.info(
"Creating a CumulusService with ram cache of %s / %s MB and %s threads",
self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusThreadCountOverride
)
self._stopEvent = threading.Event()
self._channelListener = channelListener
assert len(self._channelListener.ports) == 2
self._channelFactory = channelFactory
Runtime.initialize()
ModuleImporter.initialize()
self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
self.viewFactory
)
self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
target=self._channelListener.start
)
self.vdm = VectorDataManager.constructVDM(
callbackScheduler,
self.cumulusVectorRamCacheSizeOverride,
self.cumulusMaxRamCacheSizeOverride
)
if self.cumulusTrackTcMalloc:
logging.info(
"CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0
)
self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()
self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
viewFactory.createView(retrySeconds=10.0, numRetries=10),
callbackScheduler
)
self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)
self.deleteCumulusDiskCacheIfNecessary()
self.offlineCache = CumulusNative.DiskOfflineCache(
callbackScheduler,
self.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageMB * 1024 * 1024,
config.cumulusDiskCacheStorageFileCount
)
checkpointInterval = config.cumulusCheckpointIntervalSeconds
if checkpointInterval == 0:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None()
else:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
checkpointInterval,
1024 * 1024
)
self.cumulusWorker = self.constructCumlusWorker(
callbackScheduler,
CumulusNative.CumulusWorkerConfiguration(
self.machineId,
self.cumulusThreadCountOverride,
checkpointPolicy,
ExecutionContext.createContextConfiguration(),
diagnosticsDir or ""
),
self.vdm,
self.offlineCache,
eventHandler
)
#externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
#callbackScheduler
#)
#self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
#settings.s3InterfaceFactory,
#settings.objectStore,
#self.vdm,
#externalDatasetChannel.makeQueuelike(callbackScheduler)
#)
self.cumulusWorker.startComputations()
def initialize(self, purePythonMDSAsJson):
"""Initialize the converter assuming a set of pyfora builtins"""
try:
import pyfora.ObjectRegistry as ObjectRegistry
import ufora.FORA.python.PurePython.Converter as Converter
import ufora.FORA.python.PurePython.PyforaSingletonAndExceptionConverter as PyforaSingletonAndExceptionConverter
import ufora.native.FORA as ForaNative
import ufora.FORA.python.ModuleImporter as ModuleImporter
logging.info("Initializing the PyforaObjectConverter")
objectRegistry_[0] = ObjectRegistry.ObjectRegistry()
if purePythonMDSAsJson is None:
converter_[0] = Converter.Converter()
else:
purePythonModuleImplval = ModuleImporter.importModuleFromMDS(
ModuleDirectoryStructure.ModuleDirectoryStructure.fromJson(purePythonMDSAsJson),
"fora",
"purePython",
searchForFreeVariables=True
)
singletonAndExceptionConverter = \
PyforaSingletonAndExceptionConverter.PyforaSingletonAndExceptionConverter(
purePythonModuleImplval
)
primitiveTypeMapping = {
bool: purePythonModuleImplval.getObjectMember("PyBool"),
str: purePythonModuleImplval.getObjectMember("PyString"),
int: purePythonModuleImplval.getObjectMember("PyInt"),
float: purePythonModuleImplval.getObjectMember("PyFloat"),
type(None): purePythonModuleImplval.getObjectMember("PyNone"),
}
nativeConstantConverter = ForaNative.PythonConstantConverter(
primitiveTypeMapping
)
nativeListConverter = ForaNative.makePythonListConverter(
purePythonModuleImplval.getObjectMember("PyList")
)
nativeTupleConverter = ForaNative.makePythonTupleConverter(
purePythonModuleImplval.getObjectMember("PyTuple")
)
nativeDictConverter = ForaNative.makePythonDictConverter(
purePythonModuleImplval.getObjectMember("PyDict")
)
foraBuiltinsImplVal = ModuleImporter.builtinModuleImplVal()
converter_[0] = Converter.Converter(
nativeListConverter=nativeListConverter,
nativeTupleConverter=nativeTupleConverter,
nativeDictConverter=nativeDictConverter,
nativeConstantConverter=nativeConstantConverter,
singletonAndExceptionConverter=singletonAndExceptionConverter,
vdmOverride=ComputedValueGateway.getGateway().vdm,
purePythonModuleImplVal=purePythonModuleImplval,
foraBuiltinsImplVal=foraBuiltinsImplVal
)
except:
logging.critical("Failed to initialize the PyforaObjectConverter: %s", traceback.format_exc())
raise
def connectToRemoteWorker(self, machineId, ip, port, guid):
logging.debug("Attempting to connect to machine %s on %s:%s with guid %s",
machineId,
ip,
port,
guid)
try:
stringChannel = self._channelFactory.createChannel((ip, port))
except:
logging.error("CAN'T CONNECT TO WORKER ON %s:%s!\n"
"This may be a temporary failure but if the problem persists, "
"check the workers' network configuration and verify "
"that the machines can see each other.",
ip, port)
raise
stringChannel.write(ufora.version)
stringChannel.write(machineId.__getstate__())
#initiate the handshake
stringChannel.write(
CumulusNative.CumulusClientOrMachine.Machine(
self.machineId
).__getstate__()
)
stringChannel.write(guid.__getstate__())
logging.debug("CumulusService %s wrote handshake for %s with guid %s",
self.machineId,
machineId,
guid)
channelAsQueue = stringChannel.makeQueuelike(self.callbackScheduler)
msg = channelAsQueue.getTimeout(HANDSHAKE_TIMEOUT)
if msg is None:
logging.error("CAN'T CONNECT TO WORKER ON %s:%s!\n"
"While attempting to add worker %s with guid %s, "
"Worker %s did not receive a builtin hash message "
"during handshake.\n"
"Verify that the ufora worker is running on the remote machine.",
ip, port,
machineId, guid,
self.machineId)
return None
otherWorkersBuiltinHash = Hash.Hash(0)
otherWorkersBuiltinHash.__setstate__(msg)
builtinsAgree = otherWorkersBuiltinHash == ModuleImporter.builtinModuleImplVal().hash
if not builtinsAgree:
logging.critical("CAN'T CONNECT TO WORKER ON %s:%s!\n"
"Worker %s could not connect to Worker %s as they have "
"different builtins; former's builtin hash: %s, latter's builtin hash: "
"%s\n"
"Verify that both machines run the same ufora version.",
ip, port,
self.machineId,
machineId,
ModuleImporter.builtinModuleImplVal().hash,
otherWorkersBuiltinHash)
channelAsQueue.disconnect()
return None
return channelAsQueue
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import ufora.FORA.python.ModuleImporter as ModuleImporter
import pyfora
import ufora.native.FORA as ForaNative
builtinPythonImplVal = ModuleImporter.importModuleFromPath(
os.path.join(os.path.abspath(os.path.split(pyfora.__file__)[0]), "fora", "purePython"),
searchForFreeVariables=True
)
primitiveTypeMapping = {
bool: builtinPythonImplVal.getObjectMember("PyBool"),
str: builtinPythonImplVal.getObjectMember("PyString"),
int: builtinPythonImplVal.getObjectMember("PyInt"),
float: builtinPythonImplVal.getObjectMember("PyFloat"),
type(None): builtinPythonImplVal.getObjectMember("PyNone"),
}
defaultWrappingNativeListConverter = ForaNative.makePythonListConverter(
builtinPythonImplVal.getObjectMember("PyList")
)
defaultWrappingNativeTupleConverter = ForaNative.makePythonTupleConverter(
def doChannelHandshake(self, channel):
try:
logging.debug("Worker %s beginning channel handshake", self.machineId)
version = channel.getTimeout(HANDSHAKE_TIMEOUT)
if version is None:
logging.error(
"CAN'T ACCEPT CONNECTION!\n"
"CumulusService %s couldn't read client version within the configured timeout",
self.machineId
)
if version != ufora.version:
self.logBadUforaVersionOnChannel(version)
channel.disconnect()
return
logging.debug(
"CumulusService %s accepted connection from client with version %s",
self.machineId,
version
)
msgThatShouldBeMyOwnHash = channel.getTimeout(HANDSHAKE_TIMEOUT)
if not self.isOwnHashInHandshakeMessage(msgThatShouldBeMyOwnHash):
channel.disconnect()
return
msg = channel.getTimeout(HANDSHAKE_TIMEOUT)
if msg is None:
logging.error(
"CAN'T ACCEPT CONNECTION!\n"
"Worker %s didn't received remote machine ID during handshake",
self.machineId
)
channel.disconnect()
return
clientOrMachine = CumulusNative.CumulusClientOrMachine.Machine(
CumulusNative.MachineId(
Hash.Hash(0)
)
)
clientOrMachine.__setstate__(msg)
hashGuid = Hash.Hash(0)
msg = channel.getTimeout(HANDSHAKE_TIMEOUT)
if msg is None:
logging.error(
"CAN'T ACCEPT CONNECTION!\n"
"Worker %s didn't received handshake GUID",
self.machineId
)
channel.disconnect()
return
hashGuid.__setstate__(msg)
logging.debug(
"Worker %s accepted connection with guid %s from %s",
self.machineId,
hashGuid,
clientOrMachine
)
channel.write(
ModuleImporter.builtinModuleImplVal().hash.__getstate__()
)
with self.lock:
self._channelListener.setGroupIdForAcceptedChannel(
channel,
(clientOrMachine, hashGuid)
)
logging.debug("CumulusService %s added a channel to group %s",
self.machineId,
(clientOrMachine, hashGuid))
except:
logging.error("FAILED TO PROCESS INCOMING CONNECTION: %s", traceback.format_exc())
channel.disconnect()
# Unless required by applicable law or agreed to in writing, software
# distributed under the License is distributed on an "AS IS" BASIS,
# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
# See the License for the specific language governing permissions and
# limitations under the License.
import os
import ufora.FORA.python.ModuleImporter as ModuleImporter
import pyfora
import ufora.native.FORA as ForaNative
builtinPythonImplVal = ModuleImporter.importModuleFromPath(
os.path.join(os.path.abspath(os.path.split(pyfora.__file__)[0]), "fora",
"purePython"),
searchForFreeVariables=True)
primitiveTypeMapping = {
bool: builtinPythonImplVal.getObjectMember("PyBool"),
str: builtinPythonImplVal.getObjectMember("PyString"),
int: builtinPythonImplVal.getObjectMember("PyInt"),
float: builtinPythonImplVal.getObjectMember("PyFloat"),
type(None): builtinPythonImplVal.getObjectMember("PyNone"),
}
defaultWrappingNativeListConverter = ForaNative.makePythonListConverter(
builtinPythonImplVal.getObjectMember("PyList"))
defaultWrappingNativeTupleConverter = ForaNative.makePythonTupleConverter(
builtinPythonImplVal.getObjectMember("PyTuple"))
def __init__(self, ownAddress, channelListener, channelFactory,
eventHandler, callbackScheduler, diagnosticsDir, config,
viewFactory):
Stoppable.Stoppable.__init__(self)
#acquire a machineId randomly, using uuid
self.machineId = CumulusNative.MachineId(
Hash.Hash.sha1(str(uuid.uuid4())))
self.ownAddress = ownAddress
self.callbackScheduler = callbackScheduler
self.viewFactory = viewFactory
self.threadsStarted_ = False
self.connectedMachines = set()
self.connectingMachines = set(
) # machines we are in the process of connecting to
self.droppedMachineIds = set()
self.lock = threading.RLock()
self.cumulusMaxRamCacheSizeOverride = config.cumulusMaxRamCacheMB * 1024 * 1024
self.cumulusVectorRamCacheSizeOverride = config.cumulusVectorRamCacheMB * 1024 * 1024
self.cumulusThreadCountOverride = config.cumulusServiceThreadCount
self.cumulusTrackTcMalloc = config.cumulusTrackTcmalloc
self.reconnectPersistentCacheIndexViewThreads = []
if config.cumulusDiskCacheStorageSubdirectory is not None:
self.cumulusDiskCacheWantsDeletionOnTeardown = True
self.cumulusDiskCacheStorageDir = os.path.join(
config.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageSubdirectory)
else:
self.cumulusDiskCacheWantsDeletionOnTeardown = False
self.cumulusDiskCacheStorageDir = config.cumulusDiskCacheStorageDir
logging.info(
"Creating a CumulusService with ram cache of %s / %s MB and %s threads",
self.cumulusVectorRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusMaxRamCacheSizeOverride / 1024.0 / 1024.0,
self.cumulusThreadCountOverride)
self._stopEvent = threading.Event()
self._channelListener = channelListener
assert len(self._channelListener.ports) == 2
self._channelFactory = channelFactory
Runtime.initialize()
ModuleImporter.initialize()
self.cumulusActiveMachines = CumulusActiveMachines.CumulusActiveMachines(
self.viewFactory)
self.cumulusChannelFactoryThread = ManagedThread.ManagedThread(
target=self._channelListener.start)
self.vdm = VectorDataManager.constructVDM(
callbackScheduler, self.cumulusVectorRamCacheSizeOverride,
self.cumulusMaxRamCacheSizeOverride)
if self.cumulusTrackTcMalloc:
logging.info(
"CumulusService enabling track-tc-malloc memory with a max cache of %s MB",
self.cumulusMaxRamCacheSizeOverride / 1024 / 1024.0)
self.vdm.getMemoryManager().enableCountTcMallocMemoryAsEcMemory()
self.persistentCacheIndex = CumulusNative.PersistentCacheIndex(
viewFactory.createView(retrySeconds=10.0, numRetries=10),
callbackScheduler)
self.vdm.setPersistentCacheIndex(self.persistentCacheIndex)
self.deleteCumulusDiskCacheIfNecessary()
self.offlineCache = CumulusNative.DiskOfflineCache(
callbackScheduler, self.cumulusDiskCacheStorageDir,
config.cumulusDiskCacheStorageMB * 1024 * 1024,
config.cumulusDiskCacheStorageFileCount)
checkpointInterval = config.cumulusCheckpointIntervalSeconds
if checkpointInterval == 0:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.None ()
else:
checkpointPolicy = CumulusNative.CumulusCheckpointPolicy.Periodic(
checkpointInterval, 1024 * 1024)
self.cumulusWorker = self.constructCumlusWorker(
callbackScheduler,
CumulusNative.CumulusWorkerConfiguration(
self.machineId,
self.cumulusThreadCountOverride, checkpointPolicy,
ExecutionContext.createContextConfiguration(), diagnosticsDir
or ""), self.vdm, self.offlineCache, eventHandler)
#externalDatasetChannel = self.cumulusWorker.getExternalDatasetRequestChannel(
#callbackScheduler
#)
#self.datasetLoadService = PythonIoTaskService.PythonIoTaskService(
#settings.s3InterfaceFactory,
#settings.objectStore,
#self.vdm,
#externalDatasetChannel.makeQueuelike(callbackScheduler)
#)
self.cumulusWorker.startComputations()
def doChannelHandshake(self, channel):
try:
logging.info("CumulusService %s beginning channel handshake on channel %s", self.machineId, channel)
version = channel.getTimeout(HANDSHAKE_TIMEOUT)
if version is None:
logging.error(
"CumulusService %s couldn't read client version within the configured timeout",
self.machineId
)
if version != ufora.version:
self.logBadUforaVersionOnChannel(version)
channel.disconnect()
return
logging.info(
"CumulusService %s accepted connection from client with version %s",
self.machineId,
version
)
msgThatShouldBeMyOwnHash = channel.getTimeout(HANDSHAKE_TIMEOUT)
if not self.isOwnHashInHandshakeMessage(msgThatShouldBeMyOwnHash):
channel.disconnect()
return
msg = channel.getTimeout(HANDSHAKE_TIMEOUT)
if msg is None:
logging.error(
"CumulusService %s didn't received remote machine Id in handshake",
self.machineId
)
channel.disconnect()
return
clientOrMachine = CumulusNative.CumulusClientOrMachine.Machine(
CumulusNative.MachineId(
Hash.Hash(0)
)
)
clientOrMachine.__setstate__(msg)
hashGuid = Hash.Hash(0)
msg = channel.getTimeout(HANDSHAKE_TIMEOUT)
if msg is None:
logging.error("CumulusService %s didn't received handshake GUID", self.machineId)
channel.disconnect()
return
hashGuid.__setstate__(msg)
logging.info(
"CumulusService %s accepted connection of guid %s for client %s",
self.machineId,
hashGuid,
clientOrMachine
)
channel.write(
ModuleImporter.builtinModuleImplVal().hash.__getstate__()
)
with self.lock:
self._channelListener.setGroupIdForAcceptedChannel(
channel,
(clientOrMachine, hashGuid)
)
logging.info("CumulusService %s added a channel to group %s",
self.machineId,
(clientOrMachine, hashGuid))
except:
logging.error("Failed to process incoming connection: %s", traceback.format_exc())
channel.disconnect()
def __init__(self,
callbackScheduler,
sharedStateViewFactory,
computedValueGatewayFactory):
self.lock = threading.Lock()
self.cacheLoadEvents = {}
self.resultsById_ = {}
self.eventsById_ = {}
logging.info("created a component host")
self.graph = ComputedGraph.ComputedGraph()
logging.info("created a ComputedGraph")
Runtime.initialize()
logging.info("Runtime initialized")
ModuleImporter.initialize()
logging.info("Module importer initialized")
Fora._builtin = ForaValue.FORAValue(ModuleImporter.builtinModuleImplVal())
self.incomingObjectCache = IncomingObjectCache()
self.outgoingObjectCache = OutgoingObjectCache()
self.VDM = VectorDataManager.constructVDM(callbackScheduler)
self.VDM.setDropUnreferencedPagesWhenFull(True)
logging.info("created a VDM")
logging.info("got shared state view factory: %s", sharedStateViewFactory)
def initValueGateway():
with self.graph:
self.computedValueGateway = computedValueGatewayFactory()
self.cumulusGatewayRemote = self.computedValueGateway.cumulusGateway
def initSynchronizer():
self.synchronizer = SharedStateSynchronizer.SharedStateSynchronizer()
logging.info("created a SharedStateSynchronizer")
self.synchronizer.attachView(
sharedStateViewFactory.createView()
)
logging.info("attached shared state view.")
simultaneously(
initSynchronizer,
initValueGateway
)
self.synchronousSharedStateScope = SynchronousPropertyAccess.SynchronousPropertyAccess()
self.outstandingMessagesById = {}
self.expectedMessageId = 0
self.messageTypeHandlers = {}
self.messageTypeHandlers["Read"] = self.handleReadMessage
self.messageTypeHandlers["Assign"] = self.handleAssignMessage
self.messageTypeHandlers["Subscribe"] = self.handleSubscribeMessage
self.messageTypeHandlers["Execute"] = self.handleExecuteMessage
self.messageTypeHandlers["ServerFlushObjectIdsBelow"] = self.handleFlushObjectIds
self.pendingObjectQueue = []
self.subscriptions = Subscriptions.Subscriptions(
self.graph,
self.computedValueGateway,
self.synchronizer
)