def _deserializeEdge(key):
"""
Deserializes an edge database key
@return A tuple (sourceId, targetId, type) deserialized from the database key
>>> Edge._deserializeEdge("\\x1Fa\\x0Eb")
('a', 'b', '')
>>> Edge._deserializeEdge("\\x1Fa\\x0Fb")
('b', 'a', '')
>>> Edge._deserializeEdge("mytype\\x1Fa\\x0Eb")
('a', 'b', 'mytype')
>>> Edge._deserializeEdge("mytype\\x1Fa\\x0Fb")
('b', 'a', 'mytype')
"""
#Split the typeplanetexplorers.pathea.net
typeSeparatorIndex = key.find("\x1F")
if typeSeparatorIndex == -1:
raise ParameterException(
"Could not find type separator in edge key!")
edgeType = key[0:typeSeparatorIndex]
key = key[typeSeparatorIndex + 1:]
#Split the source and target node
outgoingIndex = key.find("\x0E")
incomingIndex = key.find("\x0F")
if outgoingIndex == -1 and incomingIndex == -1:
raise ParameterException(
"Could not find OUT or IN separator in edge key!")
isIncoming = (outgoingIndex == -1)
splitKey = (incomingIndex if isIncoming else outgoingIndex)
firstNodeIndex = key[0:splitKey]
secondNodeIndex = key[splitKey + 1:]
sourceNode = (secondNodeIndex if isIncoming else firstNodeIndex)
targetNode = (firstNodeIndex if isIncoming else secondNodeIndex)
#Create and return the tuple
return (sourceNode, targetNode, edgeType)
def _checkDictionaryForNone(dictionary):
"""Throws a parameter exception if the given dict contains any None keys or values"""
#None keys or values are not supported, they can't be mapped to binary!
# Use empty strings if neccessary.
if any(key == None for key in dictionary.keys()) is None:
raise ParameterException(
"Dictionary contains a key = None. Can't convert that to binary!"
)
if any(value == None for value in dictionary.values()) is None:
raise ParameterException(
"Dictionary contains a value = None. Can't convert that to binary!"
)
def partsync(self, flag):
"""
Set or clear the partsync flag.
"""
if type(flag) is not bool:
raise ParameterException("flag value must be a bool!")
self.partsyncFlag = flag
def __init__(self, endpoints=None, context=None):
"""
Create a new YakDB connection instance.
@param endpoints An endpoint string or list of endpoints to connect to.
New endpoints can be added dynamically.
"""
if context is None:
self.context = zmq.Context()
self.cleanupContextOnDestruct = True
else:
self.context = context
self.cleanupContextOnDestruct = False
self.socket = None
self.numConnections = 0
#Connect to the endpoints, if any
if endpoints is None:
pass
elif type(endpoints) is str or type(endpoints) is bytes:
self.connect(endpoints)
elif isinstance(connection, collections.Iterable):
[self.connect(endpoint) for endpoint in endpoints]
else: #Unknown parameter type
raise ParameterException(
"Endpoints parameter is a '%s' but expected an iterable or a string!"
% (str(type(endpoints))))
def _checkListForNone(thelist):
"""Throws a ParameterException if any of the values is None"""
#None keys or values are not supported, they can't be mapped to binary!
# Use empty strings if neccessary.
if any(key == None for key in thelist) is None:
raise ParameterException(
"Dictionary contains a key = None. Can't convert that to binary!"
)
def saveNode(self, node):
"""
Save a node and its basic attribute set into the database.
"""
if not isinstance(node, Node):
raise ParameterException("Node parameter must by a Node instance!")
self.conn.put(self.nodeTableId,
{node.id: node.basicAttributes.serialize()},
partsync=self.partsyncFlag)
def put(self,
tableNo,
valueDict,
partsync=False,
fullsync=False,
requestId=b""):
"""
Write a dictionary of key-value pairs to the connected servers.
This request can be used in REQ/REP, PUSH/PULL and PUB/SUB mode.
@param tableNo The numeric, unsigned table number to write to
@param valueDict A dictionary containing the key/value pairs to be written.
Must not contain None keys or values.
integral types are automatically mapped to signed 32-bit little-endian binary,
floating point types are mapped to signed little-endian 64-bit IEEE754 values.
If you'd like to use another binary representation, use a binary string instead!
@param partsync If set to true, subsequent reads are guaranteed to return the written values
@param fullsync If set to true, written data is synced to disk after being written.
@return True on success, else an appropriate exception will be raised
"""
#Check parameters
YakDBConnectionBase._checkParameterType(tableNo, int, "tableNo")
YakDBConnectionBase._checkParameterType(valueDict, dict, "valueDict")
#Before sending any frames, check the value dictionary for validity
#Else, the socket could be left in an inconsistent state
if len(valueDict) == 0:
raise ParameterException(
"Dictionary to be written did not contain any valid data!")
YakDBConnectionBase._checkDictionaryForNone(valueDict)
#Send header frame
self.socket.send(
YakDBConnectionBase._getWriteHeader(b"\x20", partsync, fullsync,
requestId), zmq.SNDMORE)
#Send the table number
self._sendBinary32(tableNo)
#Send key/value pairs
nextToSend = None #Needed because the last value shall be sent w/out SNDMORE
for key, value in valueDict.items():
#Send the value from the last loop iteration
if nextToSend is not None:
self.socket.send(nextToSend, zmq.SNDMORE)
#Map key & value to binary data if neccessary
key = ZMQBinaryUtil.convertToBinary(key)
value = ZMQBinaryUtil.convertToBinary(value)
#Send the key and enqueue the value
self.socket.send(key, zmq.SNDMORE)
nextToSend = value
#If nextToSend is None now, the dict didn't contain valid data
#Send the last value without SNDMORE
self.socket.send(nextToSend)
#If this is a req/rep connection, receive a reply
if self.mode is zmq.REQ:
msgParts = self.socket.recv_multipart(copy=True)
YakDBConnectionBase._checkHeaderFrame(msgParts, b'\x20')
def _checkParameterType(value, expectedType, name, allowNone=False):
"""
Raises a ParameterException if the given value does not have the given type
@param allowNone If this is set to true, a 'None' value is also allowed and doesn't throw
"""
if allowNone and value is None:
return
if type(value) is not expectedType:
raise ParameterException(
"Parameter '%s' is not a %s but a %s!" %
(name, str(expectedType), str(type(value))))
def convertToBinary(value, convertIterables=True):
"""
Given a string, float or int value, convert it to binary and return the converted value.
Ints are converted to 32-bit little-endian signed integers (uint32_t).
Floats are converted to 64-bit little-endian IEEE754 values (double).
Numeric values are assumed to be signed.
Given a list of these types, returns a list of the converted values,
if convertIterables is not set to False.
For other types, raises an exception
"""
if type(value) is bytes:
return value
elif type(value) is int:
return struct.pack('<i', value)
elif type(value) is float:
return struct.pack('<d', value)
elif type(value) is str:
return value.encode("utf-8")
#Listlike types
elif isinstance(value, collections.Iterable):
#str and bytes are also iterable, but we already handled that
if not convertIterables:
raise ParameterException(
"Value '%s' contains nested iterable or iterable conversion is disabled, can't convert!"
% str(value))
return [
ZMQBinaryUtil.convertToBinary(val, convertIterables=False)
for val in value
]
elif value is None:
raise ParameterException(
"Can't convert None values, please check the value does not contains any None values"
)
else:
raise ParameterException(
"Value '%s' of type %s is not mappable to binary. Please use a binary string for custom types."
% (value, str(type(value))))
def exists(self, tableNo, keys):
"""
Chec one or multiples values, identified by their keys, for existence in a given table.
@param tableNo The table number to read from
@param keys A list, tuple or single value.
Must only contain strings, ints or floats.
integral types are automatically mapped to signed 32-bit little-endian binary,
floating point types are mapped to signed little-endian 64-bit IEEE754 values.
If you'd like to use another binary representation, use a binary string instead.
@return A list of values, correspondent to the key order
"""
#Check parameters and create binary-string only key list
YakDBConnectionBase._checkParameterType(tableNo, int, "tableNo")
convertedKeys = []
if type(keys) is list or type(keys) is tuple:
for value in keys:
if value is None:
raise ParameterException(
"Key list contains 'None' value, not mappable to binary"
)
convertedKeys.append(ZMQBinaryUtil.convertToBinary(value))
elif (type(keys) is str) or (type(keys) is int) or (type(keys) is
float):
#We only have a single value
convertedKeys.append(ZMQBinaryUtil.convertToBinary(keys))
#Check if this connection instance is setup correctly
self._checkSingleConnection()
self._checkRequestReply()
#Send header frame
self.socket.send(b"\x31\x01\x12", zmq.SNDMORE)
#Send the table number frame
self._sendBinary32(tableNo)
#Send key list
nextToSend = None #Needed because the last value shall be sent w/out SNDMORE
for key in convertedKeys:
#Send the value from the last loop iteration
if nextToSend is not None:
self.socket.send(nextToSend, zmq.SNDMORE)
#Send the key and enqueue the value
nextToSend = key
#Send last key, without SNDMORE flag
self.socket.send(nextToSend)
#Wait for reply
msgParts = self.socket.recv_multipart(copy=True)
YakDBConnectionBase._checkHeaderFrame(msgParts, b'\x12')
#Return the data frames after mapping them to bools
processedValues = []
for msgPart in msgParts[1:]:
processedValues.append(False if msgPart == b"\x00" else True)
return processedValues
def put(self, valueDict):
"""
Write a dictionary of values to the current batch.
Note that this is slower than adding the keys one-by-one
because of the merge method currently being used
"""
if type(valueDict) is not dict:
raise ParameterException(
"Batch put valueDict parameter must be a dictionary but it's a %s"
% str(type(valueDict)))
# Merge the dicts
self.batchData = dict(self.batchData.items() + valueDict.items())
self.__checkFlush()
self.numWrites += len(valueDict)
def saveEdges(self, edges):
"""
Save multiple edges at once.
@param edges The edge list to save
"""
putDict = {}
for edge in edges:
if not isinstance(edge, Edge):
raise ParameterException(
"Edge parameter must by an Edge instance!")
serializedBasicAttrs = edge.basicAttributes.serialize()
putDict[edge.activeKey] = serializedBasicAttrs
putDict[edge.passiveKey] = serializedBasicAttrs
self.conn.put(self.edgeTableId, putDict, partsync=self.partsyncFlag)
def saveExtendedAttributes(self, entityId, attrDict):
"""
Save one or multiple extended attributes to the database
@param node The node to serialize for
@param attrDict A dictionary from key to value
"""
if type(attrDict) is not dict:
raise ParameterException(
"attrDict parameter must be a Dictionary!")
dbDict = {}
for key, value in attrDict.iteritems():
dbKey = ExtendedAttributes._serializeKey(entityId, key)
dbDict[dbKey] = value
self.conn.put(self.extendedAttributesTable,
dbDict,
partsync=self.partsyncFlag)