def _processUpdateFromBackend(self, payloadEnvelope: PayloadEnvelope):
tupleSelector: TupleSelector = payloadEnvelope.filt["tupleSelector"]
if not self._hasTupleSelector(tupleSelector):
return
cache, requiredUpdate = self._updateCache(payloadEnvelope)
if not requiredUpdate:
return
# Get / update the list of observing UUIDs
observingUuids = cache.vortexUuids & set(VortexFactory.getRemoteVortexUuids())
if not observingUuids:
return
# Create the vortexMsg
vortexMsg = payloadEnvelope.toVortexMsg()
# Send the vortex messages
for vortexUuid in observingUuids:
d = VortexFactory.sendVortexMsg(vortexMsgs=vortexMsg,
destVortexUuid=vortexUuid)
d.addErrback(vortexLogFailure, logger, consumeError=True)
def connectionLost(self, reason=connectionDone):
from vortex.VortexFactory import VortexFactory
VortexFactory._notifyOfVortexStatusChange(self._serverVortexName, online=False)
if self._sendBeatLoopingCall.running:
self._sendBeatLoopingCall.stop()
self._closed = False
def sendDeleted(self, modelSetKey: str,
traceConfigKeys: List[str]) -> None:
""" Send Deleted
Send grid updates to the client services
:param modelSetKey: The model set key
:param traceConfigKeys: A list of object buckets that have been updated
:returns: Nothing
"""
if not traceConfigKeys:
return
if peekClientName not in VortexFactory.getRemoteVortexName():
logger.debug("No clients are online to send the doc chunk to, %s",
traceConfigKeys)
return
payload = Payload(filt=copy(clientTraceConfigUpdateFromServerFilt))
payload.filt[plDeleteKey] = True
payload.tuples = dict(modelSetKey=modelSetKey,
traceConfigKeys=traceConfigKeys)
payloadEnvelope = yield payload.makePayloadEnvelopeDefer()
vortexMsg = yield payloadEnvelope.toVortexMsgDefer()
try:
VortexFactory.sendVortexMsg(vortexMsg,
destVortexName=peekClientName)
except Exception as e:
logger.exception(e)
def _notifyOfTupleUpdateInMainOne(self, tupleSelector: TupleSelector):
tsStr = tupleSelector.toJsonStr()
if tsStr not in self._observerDataByTupleSelector:
return
# Filter out the offline observables
onlineUuids = set(VortexFactory.getRemoteVortexUuids())
observers = self._observerDataByTupleSelector[tsStr].observers
for od in list(observers):
if od.vortexUuid not in onlineUuids:
observers.remove(od)
# Get / update the list of observing UUIDs
if not observers:
del self._observerDataByTupleSelector[tsStr]
return
# Create the vortexMsg
filt = dict(tupleSelector=tupleSelector)
filt.update(self._filt)
vortexMsg = yield self._createVortexMsg(filt, tupleSelector)
# We can have multiple Observable clients on the one vortex, so make sure
# we only send one message for these.
destVortexUuids = set([od.vortexUuid for od in observers])
# Send the vortex messages
for destVortexUuid in destVortexUuids:
d = VortexFactory.sendVortexMsg(vortexMsgs=vortexMsg,
destVortexUuid=destVortexUuid)
d.addErrback(vortexLogFailure, logger)
def _nameAndUuidReceived(self, name, uuid):
from vortex.VortexFactory import VortexFactory
VortexFactory._notifyOfVortexStatusChange(name, online=True)
self._producer.setRemoteVortexName(self._serverVortexName)
if self._vortexClient:
self._vortexClient._setNameAndUuid(name=self._serverVortexName,
uuid=self._serverVortexUuid)
def sendModelUpdate(self,
objId,
vortexUuid=None,
session=None,
obj=None,
**kwargs):
session = session if session else self._getSession()
pl = self._retrieve(session, objId, self._payloadFilter, obj=obj)
pl.filt.update(self._payloadFilter)
pl.filt[plIdKey] = objId
VortexFactory.sendVortexMsg(pl.toVortexMsg(),
destVortexUuid=vortexUuid)
def _processCall(self, payloadEnvelope: PayloadEnvelope, vortexName,
sendResponse, *args, **kwargs):
""" Process
Process the incoming RPC call payloads.
"""
# If the sending vortex, is local, then ignore it, RPC can not be called locally
if VortexFactory.isVortexNameLocal(vortexName):
logger.warning(
"Received RPC call to %s, from local vortex %s, ignoring it",
self.__funcName, vortexName)
return
# Apply the "allow" logic
if self.__acceptOnlyFromVortex and vortexName not in self.__acceptOnlyFromVortex:
logger.debug(
"Call from non-accepted vortex %s, allowing only from %s",
vortexName, str(self.__acceptOnlyFromVortex))
return
# Get the args tuple
payload = yield payloadEnvelope.decodePayloadDefer()
argsTuple = payload.tuples[0]
assert isinstance(
argsTuple,
_VortexRPCArgTuple), ("argsTuple is not an instance of %s" %
_VortexRPCArgTuple)
logger.debug("Received RPC call for %s", self.__funcName)
# Call the method and setup the callbacks
result = yield self.callLocally(argsTuple.args, argsTuple.kwargs)
yield self._processCallCallback(result, sendResponse,
payloadEnvelope.filt)
def sendCreatedOrUpdatedUpdates(self, modelSetKey: str,
traceConfigKeys: List[str]) -> None:
""" Send Create or Updated Updates
Send grid updates to the client services
:param modelSetKey: The model set key
:param traceConfigKeys: A list of the keys updated
:returns: Nothing
"""
if not traceConfigKeys:
return
if peekClientName not in VortexFactory.getRemoteVortexName():
logger.debug(
"No clients are online to send the trace configs to, %s",
traceConfigKeys)
return
def send(vortexMsg: bytes):
if vortexMsg:
VortexFactory.sendVortexMsg(vortexMsg,
destVortexName=peekClientName)
d: Deferred = self._loadTraceConfigs(modelSetKey, traceConfigKeys)
d.addCallback(send)
d.addErrback(self._sendErrback, traceConfigKeys)
def _processForProxy(self, payloadEnvelope: PayloadEnvelope,
vortexName: str,
sendResponse: SendVortexMsgResponseCallable,
**kwargs):
# Keep a copy of the incoming filt, in case they are using PayloadResponse
responseFilt = copy(payloadEnvelope.filt)
# Track the response, log an error if it fails
# 5 Seconds is long enough.
# VortexJS defaults to 10s, so we have some room for round trip time.
pr = PayloadResponse(
payloadEnvelope,
timeout=PayloadResponse.TIMEOUT - 5, # 5 seconds less
resultCheck=False,
logTimeoutError=False)
# This is not a lambda, so that it can have a breakpoint
def reply(payloadEnvelope: PayloadEnvelope):
payloadEnvelope.filt = responseFilt
d: Deferred = payloadEnvelope.toVortexMsgDefer()
d.addCallback(sendResponse)
return d
pr.addCallback(reply)
pr.addCallback(
lambda _: logger.debug("Received action response from server"))
pr.addErrback(self.__handlePrFailure, payloadEnvelope, sendResponse)
vortexMsg = yield payloadEnvelope.toVortexMsgDefer()
try:
yield VortexFactory.sendVortexMsg(
vortexMsgs=vortexMsg, destVortexName=self._proxyToVortexName)
except Exception as e:
logger.exception(e)
def startListening():
from peek_platform import PeekPlatformConfig
from peek_storage._private.service.StorageSiteResource import setupPlatformSite
from peek_storage._private.service.StorageSiteResource import platformSiteRoot
setupPlatformSite()
platformCfg = PeekPlatformConfig.config.platformHttpServer
setupSite("Peek Platform Data Exchange",
platformSiteRoot,
portNum=platformCfg.sitePort,
enableLogin=False,
redirectFromHttpPort=platformCfg.redirectFromHttpPort,
sslBundleFilePath=platformCfg.sslBundleFilePath)
VortexFactory.createTcpServer(
name=PeekPlatformConfig.componentName,
port=PeekPlatformConfig.config.peekStorageVortexTcpPort)
def setupVortexServer(portNum=8345, wsPortNum=8344):
""" Setup Site
Sets up the web site to listen for connections and serve the site.
Supports customisation of resources based on user details
@return: Port object
"""
defer.setDebugging(True)
# Register the test tuple
from vortex.test import TestTuple
TestTuple.__unused = False # Crap code
from vortex.test import VortexJSTupleLoaderTestHandler
VortexJSTupleLoaderTestHandler.__unused = False # Crap code
rootResource = TestRootResource()
VortexFactory.createServer("default", rootResource)
VortexFactory.createWebsocketServer("default", wsPortNum)
# rootResource.putChild(b"vortex", VortexResource())
site = server.Site(rootResource)
site.protocol = HTTPChannel
port = reactor.listenTCP(portNum, site).port
# ip = subprocess.getoutput("/sbin/ifconfig").split("\n")[1].split()[1][5:]
logger.info('VortexServer test is alive and listening on port %s', port)
logger.info('VortexServerWebsocket test is alive and listening on port %s',
wsPortNum)
logger.debug("Logging debug messages enabled")
NotifyTestTimer.startTupleUpdateNotifyer()
return port
def __cacheCheck(self):
currentVortexUuids = set(VortexFactory.getRemoteVortexUuids())
for ts, cache in list(self.__cache.items()):
cache.vortexUuids = cache.vortexUuids & currentVortexUuids
if cache.vortexUuids or cache.subject.observers:
cache.resetTearDown()
elif cache.isReadyForTearDown():
logger.debug("Cleaning cache for %s", cache.tupleSelector)
self._sendUnsubscribeToServer(cache.tupleSelector)
del self.__cache[ts]
else:
cache.markForTearDown()
def _filterOutOfflineVortexes(self):
# TODO, Change this to observe offline vortexes
# This depends on the VortexFactory offline observable implementation.
# Which is incomplete at this point :-|
vortexUuids = set(VortexFactory.getRemoteVortexUuids())
vortexUuidsToRemove = set(
self._observedGridKeysByVortexUuid) - vortexUuids
if not vortexUuidsToRemove:
return
for vortexUuid in vortexUuidsToRemove:
del self._observedGridKeysByVortexUuid[vortexUuid]
self._rebuildStructs()
def start(self, funcSelf=None):
""" Start
If this is a class method, then bind the function to the
object passed in by bindToSelf.
:param funcSelf: The object to bind the class instance methods self to.
"""
if VortexFactory.isVortexNameLocal(self.__listeningVortexName):
self.__ep = PayloadEndpoint(self._filt, self._processCall)
logger.debug("RPC %s listening", self.__funcName)
else:
logger.error(
"Ignoring request to start listening for RPC %s "
"as vortex name %s is not local", self.__funcName,
self.__listeningVortexName)
self.__funcSelf = funcSelf
return self
import logging
from twisted.internet import reactor
from vortex.VortexFactory import VortexFactory
from vortex.rpc.RPC import vortexRPC
from vortex.rpc.RPCTest import myRemoteAddMethod
logger = logging.getLogger(__name__)
logging.basicConfig()
if __name__ == '__main__':
VortexFactory.createTcpServer("listenVortexName", 10101)
reactor.callLater(0, logger.info, "RPCTest server running")
reactor.callLater(0, myRemoteAddMethod.start)
reactor.run()
def send(vortexMsg: bytes):
if vortexMsg:
VortexFactory.sendVortexMsg(vortexMsg,
destVortexName=peekClientName)
def _sendRequestToServer(self, payloadEnvelope: PayloadEnvelope):
payloadEnvelope.filt["observerName"] = self._observerName
d = VortexFactory.sendVortexMsg(vortexMsgs=payloadEnvelope.toVortexMsg(),
destVortexName=self._proxyToVortexName)
d.addErrback(vortexLogFailure, logger, consumeError=True)
def connect():
yield VortexFactory.createTcpClient("sendVortexName", "127.0.0.1", 10101)
reactor.callLater(2, callRpc)
def _sendRequestToServer(self, payload):
payload.filt.update(self._sendFilt)
d = VortexFactory.sendVortexMsg(vortexMsgs=payload.toVortexMsg(),
destVortexName=self._destVortexName)
d.addErrback(vortexLogFailure, logger, consumeError=True)
