def makeVortexMsg(self, filt: dict,
tupleSelector: TupleSelector) -> Deferred:
deviceToken = tupleSelector.selector["deviceToken"]
session = self._dbSessionCreator()
try:
userLoggedIn = session.query(UserLoggedIn) \
.filter(UserLoggedIn.deviceToken == deviceToken) \
.one()
internalUserTuple = session.query(InternalUserTuple) \
.filter(InternalUserTuple.userName == userLoggedIn.userName) \
.one()
userDetails = UserListItemTuple(
userId=internalUserTuple.userName,
displayName=internalUserTuple.userTitle)
except NoResultFound:
userDetails = None
finally:
session.close()
tuples = [
UserLoggedInTuple(deviceToken=deviceToken, userDetails=userDetails)
]
payloadEnvelope = Payload(filt=filt,
tuples=tuples).makePayloadEnvelope()
vortexMsg = payloadEnvelope.toVortexMsg()
return vortexMsg
def sendDeleted(self, modelSetKey: str,
traceConfigKeys: List[str]) -> None:
""" Send Deleted
Send grid updates to the client services
:param modelSetKey: The model set key
:param traceConfigKeys: A list of object buckets that have been updated
:returns: Nothing
"""
if not traceConfigKeys:
return
if peekClientName not in VortexFactory.getRemoteVortexName():
logger.debug("No clients are online to send the doc chunk to, %s",
traceConfigKeys)
return
payload = Payload(filt=copy(clientTraceConfigUpdateFromServerFilt))
payload.filt[plDeleteKey] = True
payload.tuples = dict(modelSetKey=modelSetKey,
traceConfigKeys=traceConfigKeys)
payloadEnvelope = yield payload.makePayloadEnvelopeDefer()
vortexMsg = yield payloadEnvelope.toVortexMsgDefer()
try:
VortexFactory.sendVortexMsg(vortexMsg,
destVortexName=peekClientName)
except Exception as e:
logger.exception(e)
def _loadInPg(cls, plpy, filt: dict, tupleSelectorStr: str):
tupleSelector = TupleSelector.fromJsonStr(tupleSelectorStr)
tuples = cls.loadTuples(plpy, tupleSelector)
payloadEnvelope = Payload(filt=filt, tuples=tuples).makePayloadEnvelope()
vortexMsg = payloadEnvelope.toVortexMsg()
return vortexMsg.decode()
def testSerialiseDeserialise(self):
payload = Payload(filt={"key": "PayloadResponseTest"})
d = PayloadResponse(payload, timeout=0)
d.addErrback(lambda _: True) # Handle the errback
vortexMsg = payload.toVortexMsg()
Payload().fromVortexMsg(vortexMsg)
def makeVortexMsg(self, filt: dict,
tupleSelector: TupleSelector) -> Union[Deferred, bytes]:
modelSetKey = tupleSelector.selector["modelSetKey"]
keys = tupleSelector.selector["keys"]
keysByChunkKey = defaultdict(list)
foundDocuments: List[DocumentTuple] = []
for key in keys:
keysByChunkKey[makeChunkKey(modelSetKey, key)].append(key)
for chunkKey, subKeys in keysByChunkKey.items():
chunk: DocDbEncodedChunk = self._cacheHandler.encodedChunk(
chunkKey)
if not chunk:
logger.warning("Document chunk %s is missing from cache",
chunkKey)
continue
docsByKeyStr = Payload().fromEncodedPayload(
chunk.encodedData).tuples[0]
docsByKey = json.loads(docsByKeyStr)
for subKey in subKeys:
if subKey not in docsByKey:
logger.warning(
"Document %s is missing from index, chunkKey %s",
subKey, chunkKey)
continue
# Reconstruct the data
objectProps: {} = json.loads(docsByKey[subKey])
# Get out the object type
thisDocumentTypeId = objectProps['_dtid']
del objectProps['_dtid']
# Get out the object type
thisModelSetId = objectProps['_msid']
del objectProps['_msid']
# Create the new object
newObject = DocumentTuple()
foundDocuments.append(newObject)
newObject.key = subKey
newObject.modelSet = DocDbModelSet(id=thisModelSetId)
newObject.documentType = DocDbDocumentTypeTuple(
id=thisDocumentTypeId)
newObject.document = objectProps
# Create the vortex message
return Payload(
filt, tuples=foundDocuments).makePayloadEnvelope().toVortexMsg()
def sendChunk(toSend):
if not toSend and not cacheAll:
return
payload = Payload(filt=filt, tuples=toSend)
d: Deferred = payload.makePayloadEnvelopeDefer(compressionLevel=2)
d.addCallback(
lambda payloadEnvelope: payloadEnvelope.toVortexMsgDefer())
d.addCallback(sendResponse)
d.addErrback(vortexLogFailure, logger, consumeError=True)
def makeVortexMsg(self, filt: dict,
tupleSelector: TupleSelector) -> Union[Deferred, bytes]:
tuple_ = DocumentUpdateDateTuple()
tuple_.updateDateByChunkKey = {
key:self._cacheHandler.encodedChunk(key).lastUpdate
for key in self._cacheHandler.encodedChunkKeys()
}
payload = Payload(filt, tuples=[tuple_])
payloadEnvelope = yield payload.makePayloadEnvelopeDefer()
vortexMsg = yield payloadEnvelope.toVortexMsg()
return vortexMsg
def makeVortexMsg(self, filt: dict, tupleSelector: TupleSelector) -> Deferred:
dbSession = self._dbSessionCreator()
try:
tuples = dbSession.query(EventDBModelSetTable).all()
finally:
dbSession.close()
payloadEnvelope = Payload(filt=filt, tuples=tuples).makePayloadEnvelope()
vortexMsg = payloadEnvelope.toVortexMsg()
return vortexMsg
def _retrieve(self,
session,
filtId,
payloadFilt,
obj=None,
**kwargs) -> PayloadEnvelope:
ph = obj if obj else self._getDeclarativeById(session, filtId)
payload = Payload()
payload.tuples = [ph] if ph else []
self._ext.afterRetrieve(payload.tuples, session, payloadFilt)
return payload.makePayloadEnvelope()
def pushAction(self,
tupleAction: TupleActionABC,
additionalFilt=None) -> Deferred:
""" Push Action
This pushes the action, either locally or to the server, depending on the
implementation.
If pushed locally, the promise will resolve when the action has been saved.
If pushed directly to the server, the promise will resolve when the server has
responded.
:param tupleAction The tuple action to send to the remote end
:param additionalFilt: Any extra filt that may be required to send with the
payload.
"""
filt = copy(self._filt)
if additionalFilt:
filt.update(additionalFilt)
d = Payload(filt=filt, tuples=[tupleAction]).makePayloadEnvelopeDefer()
d.addCallback(lambda payloadEnvelope: PayloadResponse(
payloadEnvelope, destVortexName=self._destVortexName))
# Convert the data to TupleAction
d.addCallback(
lambda payloadEnvelope: payloadEnvelope.decodePayloadDefer())
d.addCallback(lambda payload: payload.tuples)
return d
def process(self, payload, vortexUuid=None, userAccess=None, **kwargs):
from peek_server.PeekServerConfig import peekServerConfig
from peek_server.server.auth import AuthValue
from peek_server.storage.Setting import internalSetting, CAPABILITIES_KEY
result = None
from peek_server.storage import dbConn
session = dbConn.ormSession
# Force capabilities reload on page load
peekServerConfig._capabilities = None
# This is an update
if payload.tuples:
dataWrapTuple = payload.tuples[0]
self._capabilities = None
try:
AuthValue.loadCapabilities(self, dataWrapTuple.data)
except Exception as e:
pass
if self._capabilities is None:
result = {
"success": False,
"message":
"The license entered is not valid for this server"
}
vortexSendPayload(Payload(filt=self._payloadFilter,
result=result),
vortexUuid=vortexUuid)
return
peekServerConfig._capabilities = self._capabilities
internalSetting()[CAPABILITIES_KEY] = dataWrapTuple.data
session.commit()
result = {
"success": True,
"message": "The license was successfully loaded"
}
dataWrapTuple = DataWrapTuple()
dataWrapTuple.data = peekServerConfig.capabilities
vortexSendPayload(Payload(filt=self._payloadFilter,
tuples=[dataWrapTuple],
result=result),
vortexUuid=vortexUuid)
navbarDataHandler.sendModelUpdate(vortexUuid=vortexUuid,
userAccess=userAccess)
def _callback(self, result, replyFilt: dict, tupleName: str,
sendResponse: SendVortexMsgResponseCallable):
if not isinstance(result, list):
result = [result]
payload = Payload(filt=replyFilt, tuples=result)
payloadEnvelope = yield payload.makePayloadEnvelopeDefer()
vortexMsg = yield payloadEnvelope.toVortexMsgDefer()
try:
yield sendResponse(vortexMsg)
except Exception as e:
logger.error("Failed to send TupleAction response for %s", tupleName)
logger.exception(e)
def makeVortexMsg(self, filt: dict,
tupleSelector: TupleSelector) -> Union[Deferred, bytes]:
modelSetKey = tupleSelector.selector["modelSetKey"]
keys = tupleSelector.selector["keys"]
keysByChunkKey = defaultdict(list)
foundLocationIndexes: List[LocationIndexTuple] = []
for key in keys:
keysByChunkKey[dispKeyHashBucket(modelSetKey, key)].append(key)
for chunkKey, subKeys in keysByChunkKey.items():
chunk: EncodedLocationIndexTuple = self._locationCache.encodedChunk(chunkKey)
if not chunk:
logger.warning("Location index chunk %s is missing from cache", chunkKey)
continue
jsonStr = Payload().fromEncodedPayload(chunk.encodedLocationIndexTuple).tuples[0].jsonStr
locationsByKey = {i[0]: i[1:] for i in json.loads(jsonStr)}
for subKey in subKeys:
if subKey not in locationsByKey:
logger.warning(
"LocationIndex %s is missing from index, chunkKey %s",
subKey, chunkKey
)
continue
# Reconstruct the data
for locationJsonStr in locationsByKey[subKey]:
dispLocation = DispKeyLocationTuple.fromLocationJson(locationJsonStr)
foundLocationIndexes.append(dispLocation)
# Populate the coord set key
coordSet = self._coordSetCache.coordSetForId(dispLocation.coordSetId)
if coordSet is None:
logger.warning("Can not find coordSet with ID %s",
dispLocation.coordSetId)
continue
dispLocation.coordSetKey = coordSet.key
# Create the vortex message
return Payload(filt,
tuples=foundLocationIndexes).makePayloadEnvelope().toVortexMsg()
def _buildIndex(chunkKeys) -> Dict[str, bytes]:
session = CeleryDbConn.getDbSession()
try:
indexQry = (session.query(
DocDbDocument.chunkKey, DocDbDocument.key,
DocDbDocument.documentJson).filter(
DocDbDocument.chunkKey.in_(chunkKeys)).order_by(
DocDbDocument.key).yield_per(1000).all())
# Create the ChunkKey -> {id -> packedJson, id -> packedJson, ....]
packagedJsonByObjIdByChunkKey = defaultdict(dict)
for item in indexQry:
packagedJsonByObjIdByChunkKey[item.chunkKey][
item.key] = item.documentJson
encPayloadByChunkKey = {}
# Sort each bucket by the key
for chunkKey, packedJsonByKey in packagedJsonByObjIdByChunkKey.items():
tuples = json.dumps(packedJsonByKey, sort_keys=True)
# Create the blob data for this index.
# It will be docDbed by a binary sort
encPayloadByChunkKey[chunkKey] = Payload(
tuples=tuples).toEncodedPayload()
return encPayloadByChunkKey
finally:
session.close()
def createOrUpdateDocuments(self, documentsEncodedPayload: bytes) -> None:
startTime = datetime.now(pytz.utc)
# Decode arguments
newDocuments: List[ImportDocumentTuple] = (
Payload().fromEncodedPayload(documentsEncodedPayload).tuples)
_validateNewDocuments(newDocuments)
modelSetIdByKey = _loadModelSets()
# Do the import
try:
documentByModelKey = defaultdict(list)
for doc in newDocuments:
documentByModelKey[doc.modelSetKey].append(doc)
for modelSetKey, docs in documentByModelKey.items():
modelSetId = modelSetIdByKey.get(modelSetKey)
if modelSetId is None:
modelSetId = _makeModelSet(modelSetKey)
modelSetIdByKey[modelSetKey] = modelSetId
docTypeIdsByName = _prepareLookups(docs, modelSetId)
_insertOrUpdateObjects(docs, modelSetId, docTypeIdsByName)
logger.info("Imported %s Documents in %s", len(newDocuments),
datetime.now(pytz.utc) - startTime)
except Exception as e:
logger.debug("Retrying import docDb objects, %s", e)
raise self.retry(exc=e, countdown=3)
def makeVortexMsg(self, filt: dict,
tupleSelector: TupleSelector) -> Union[Deferred, bytes]:
userId = tupleSelector.selector["userId"]
session = self._ormSessionCreator()
try:
tasks = (session.query(Task).filter(Task.userId == userId).options(
subqueryload(Task.actions)).all())
session.expunge_all()
# Remove the data we don't want in the UI
for task in tasks:
task.onDeliveredPayloadEnvelope = None
task.onCompletedPayloadEnvelope = None
task.onDeletedPayloadEnvelope = None
task.onDialogConfirmPayloadEnvelope = None
for action in task.actions:
action.onActionPayloadEnvelope = None
# Create the vortex message
msg = Payload(filt,
tuples=tasks).makePayloadEnvelope().toVortexMsg()
finally:
session.close()
return msg
def createOrUpdateSegments(self, segmentEncodedPayload: bytes) -> None:
# Decode arguments
newSegments: List[GraphDbImportSegmentTuple] = (
Payload().fromEncodedPayload(segmentEncodedPayload).tuples)
_validateNewSegments(newSegments)
modelSetIdByKey = _loadModelSets()
# Do the import
try:
segmentByModelKey = defaultdict(list)
for segment in newSegments:
segmentByModelKey[segment.modelSetKey].append(segment)
for modelSetKey, segments in segmentByModelKey.items():
modelSetId = modelSetIdByKey.get(modelSetKey)
if modelSetId is None:
modelSetId = _makeModelSet(modelSetKey)
modelSetIdByKey[modelSetKey] = modelSetId
_insertOrUpdateObjects(segments, modelSetId, modelSetKey)
except Exception as e:
logger.debug("Retrying import graphDb objects, %s", e)
raise self.retry(exc=e, countdown=3)
def makeVortexMsg(self, filt: dict,
tupleSelector: TupleSelector) -> Deferred:
# TODO HACK - Hard coded for mobile users
mobileGroup = yield self._mobileGroup()
users = yield self._ourApi.infoApi.users(groupNames=[mobileGroup])
tuples = []
for userDetails in users:
tuples.append(
UserListItemTuple(userId=userDetails.userName,
displayName=userDetails.userTitle))
payload = Payload(filt=filt, tuples=tuples)
paylodEnvelope = yield payload.makePayloadEnvelopeDefer()
vortexMsg = yield paylodEnvelope.toVortexMsgDefer()
return vortexMsg
def makeVortexMsg(self, filt: dict,
tupleSelector: TupleSelector) -> Union[Deferred, bytes]:
session = self._ormSessionCreator()
try:
Lookup = TUPLE_TYPES_BY_NAME[tupleSelector.name]
if Lookup == DispLevel:
all = (session.query(DispLevel).options(
joinedload(DispLevel.coordSet).joinedload(
ModelCoordSet.modelSet)).all())
for item in all:
item.data = {"modelSetKey": item.coordSet.modelSet.key}
else:
all = (session.query(Lookup).options(
joinedload(Lookup.modelSet)).all())
for item in all:
item.data = {"modelSetKey": item.modelSet.key}
return Payload(filt,
tuples=all).makePayloadEnvelope().toVortexMsg()
finally:
session.close()
def _delete(self, session, tuples, filtId, payloadFilt) -> PayloadEnvelope:
self._ext.beforeDelete(tuples, session, payloadFilt)
if len(tuples):
phIds = [t.id for t in tuples]
else:
phIds = [filtId]
for phId in phIds:
ph = self._getDeclarativeById(session, phId)
try:
# Try to iterate it
for item in iter(ph):
session.delete(item)
except TypeError:
# If it's not an iterator
if ph is not None:
session.delete(ph)
session.commit()
returnTuples: List[Tuple] = []
if self._retreiveAll:
returnTuples = self.createDeclarative(session, payloadFilt)
self._ext.afterDeleteCommit(tuples, session, payloadFilt)
return Payload(tuples=returnTuples).makePayloadEnvelope(result=True)
def createOrUpdateBranches(self, importBranchesEncodedPayload: bytes) -> None:
""" Convert Import Branch Tuples
This method takes import branch tuples, and converts them to
branch format used throughout the diagram plugin.
(Thats the packed JSON wrapped by an accessor class)
"""
# Decode importBranches payload
importBranches: List[ImportBranchTuple] = (
Payload().fromEncodedPayload(importBranchesEncodedPayload).tuples)
# Validate the input importBranches
_validateNewBranchIndexs(importBranches)
# Do the import
groupedBranches = _convertImportBranchTuples(importBranches)
startTime = datetime.now(pytz.utc)
dbSession = CeleryDbConn.getDbSession()
engine = CeleryDbConn.getDbEngine()
conn = engine.connect()
transaction = conn.begin()
try:
for (modelSetKey, modelSetId,
coordSetId), branches in groupedBranches.items():
_insertOrUpdateBranches(conn, modelSetKey, modelSetId, branches)
newDisps, dispIdsToCompile = _convertBranchDisps(branches)
# NO TRANSACTION
# Bulk load the Disps
_bulkInsertDisps(engine, newDisps)
# Queue the compiler
DispCompilerQueueController.queueDispIdsToCompileWithSession(
dispIdsToCompile, conn)
transaction.commit()
dbSession.commit()
logger.debug(
"Completed importing %s branches for coordSetId %s in %s",
len(branches), coordSetId,
(datetime.now(pytz.utc) - startTime))
except Exception as e:
dbSession.rollback()
transaction.rollback()
logger.debug("Retrying createOrUpdateBranches, %s", e)
logger.exception(e)
raise self.retry(exc=e, countdown=3)
finally:
dbSession.close()
conn.close()
def createOrUpdateTraceConfigs(self, traceConfigEncodedPayload: bytes
) -> Dict[str, List[str]]:
# Decode arguments
newTraceConfigs: List[GraphDbTraceConfigTuple] = (
Payload().fromEncodedPayload(traceConfigEncodedPayload).tuples
)
_validateNewTraceConfigs(newTraceConfigs)
modelSetIdByKey = _loadModelSets()
# Do the import
try:
insertedOrCreated: Dict[str, List[str]] = defaultdict(list)
traceConfigByModelKey = defaultdict(list)
for traceConfig in newTraceConfigs:
traceConfigByModelKey[traceConfig.modelSetKey].append(traceConfig)
insertedOrCreated[traceConfig.modelSetKey].append(traceConfig.key)
for modelSetKey, traceConfigs in traceConfigByModelKey.items():
modelSetId = modelSetIdByKey.get(modelSetKey)
if modelSetId is None:
modelSetId = _makeModelSet(modelSetKey)
modelSetIdByKey[modelSetKey] = modelSetId
_insertOrUpdateObjects(traceConfigs, modelSetId)
return insertedOrCreated
except Exception as e:
logger.debug("Retrying import graphDb objects, %s", e)
raise self.retry(exc=e, countdown=3)
def getSegmentKeys(self, modelSetKey: str, vertexKey: str) -> List[str]:
chunkKey = makeChunkKeyForItemKey(modelSetKey, vertexKey)
# noinspection PyTypeChecker
chunk: ItemKeyIndexEncodedChunk = self.encodedChunk(chunkKey)
if not chunk:
logger.warning("ItemKeyIndex chunk %s is missing from cache", chunkKey)
return []
resultsByKeyStr = Payload().fromEncodedPayload(chunk.encodedData).tuples[0]
resultsByKey = json.loads(resultsByKeyStr)
if vertexKey not in resultsByKey:
logger.warning(
"ItemKey %s is missing from index, chunkKey %s",
vertexKey, chunkKey
)
return []
packedJson = resultsByKey[vertexKey]
segmentKeys = json.loads(packedJson)
return segmentKeys
def __call__(self, *args, **kwargs):
""" Call
"""
yesMainThread()
try:
# FAKE Exception so we can raise a better stack trace later
raise Exception()
except:
stack = sys.exc_info()[2]
logger.debug("Calling RPC for %s", self.__funcName)
payloadEnvelope = yield (Payload(
filt=copy(self._filt),
tuples=[_VortexRPCArgTuple(args=args, kwargs=kwargs)
]).makePayloadEnvelopeDefer(compressionLevel=4))
pr = PayloadResponse(payloadEnvelope,
timeout=self.__timeoutSeconds,
resultCheck=False,
logTimeoutError=False,
destVortexName=self.__listeningVortexName)
# Delete the payload, we don't need to keep it in memory while we
# get the result.
del payloadEnvelope
pr.addCallback(self._processResponseCallback, stack)
pr.addErrback(self._processResponseErrback, stack)
val = yield pr
return val
def makeVortexMsg(self, filt: dict,
tupleSelector: TupleSelector) -> Union[Deferred, bytes]:
tuples = [self._statusController.status]
payloadEnvelope = yield Payload(filt, tuples=tuples).makePayloadEnvelopeDefer()
vortexMsg = yield payloadEnvelope.toVortexMsgDefer()
return vortexMsg
def _buildIndex(chunkKeys) -> Dict[str, bytes]:
session = CeleryDbConn.getDbSession()
try:
indexQry = (session.query(
BranchIndex.chunkKey,
BranchIndex.key, BranchIndex.packedJson).filter(
BranchIndex.chunkKey.in_(chunkKeys)).order_by(
BranchIndex.key).yield_per(1000).all())
# Create the ChunkKey -> {key -> packedJson, key -> packedJson, ....]
packagedJsonsByObjKeyByChunkKey = defaultdict(
lambda: defaultdict(list))
for item in indexQry:
packagedJsonsByObjKeyByChunkKey[item.chunkKey][item.key].append(
item.packedJson)
encPayloadByChunkKey = {}
# Sort each bucket by the key
for chunkKey, packedJsonsByKey in packagedJsonsByObjKeyByChunkKey.items(
):
tuples = json.dumps(packedJsonsByKey, sort_keys=True)
# Create the blob data for this index.
# It will be index-blueprint by a binary sort
encPayloadByChunkKey[chunkKey] = Payload(
tuples=tuples).toEncodedPayload()
return encPayloadByChunkKey
finally:
session.close()
def _payloadEncodeDecode(self, size):
origPayload = makeTestPayloadA(size)
origPayload.date = None
encodedPayload = origPayload.toEncodedPayload()
payload = Payload().fromEncodedPayload(encodedPayload)
self.assertEqual(payload.tuples[0], origPayload.tuples[0])
def makeVortexMsg(self, filt: dict,
tupleSelector: TupleSelector) -> Union[Deferred, bytes]:
tuples = self._coordSetCacheController.coordSets
payloadEnvelope = yield Payload(
filt, tuples=tuples).makePayloadEnvelopeDefer()
vortexMsg = yield payloadEnvelope.toVortexMsgDefer()
return vortexMsg
def testPayloadToFromEncodedPayload10mb(self):
# Create Payload
origPayload = makeTestPayloadA()
origPayload.tuples = ['1234567890' * 1024 ** 2]
encodedPayload = origPayload.toEncodedPayload()
payload = Payload().fromEncodedPayload(encodedPayload)
self.assertEqual(payload.tuples[0], origPayload.tuples[0])
def testClassStaysInScope(self):
filt = {'key': 'unittest', 'This matches': 555}
payload = Payload()
payload.filt = filt
inst = PayloadEndpointPyTestAssignPayload(self)
PayloadEndpoint(filt, inst.process)
PayloadIO().process(payloadEnvelope=payload.makePayloadEnvelope(),
vortexUuid='test',
vortexName='test',
httpSession=None,
sendResponse=lambda x: x)
self.assertEqual(
self.deliveredPayloadEnvelope, payload,
'PayloadIO/PayloadEndpoint unmatched value test error')
