def test_memory_growth_transactions_changing_values(self):
connectionState = DatabaseConnectionState()
m0 = currentMemUsageMb()
for i in range(20000):
connectionState.incomingTransaction(
i,
{ObjectFieldId(objId=0, fieldId=0, isIndexValue=False): b" " * i},
{IndexId(fieldId=0, indexValue=b" " * i): (0,)},
{IndexId(fieldId=0, indexValue=b" " * (i-1)): (0,)} if i > 0 else {},
)
self.assertLess(currentMemUsageMb() - m0, 1)
def _parseSubscriptionMsg(self, channel, msg):
schema_name = msg.schema
definition = channel.definedSchemas.get(schema_name)
assert definition is not None, "can't subscribe to a schema we don't know about!"
assert msg.typename is not None
typename = msg.typename
assert typename in definition, (
"Can't subscribe to a type we didn't define in the schema: %s not in %s"
% (typename, list(definition)))
typedef = definition[typename]
if msg.fieldname_and_value is None:
field, val = " exists", indexValueFor(bool, True)
else:
field, val = msg.fieldname_and_value
if field == '_identity':
# single value identities are encoded as integers
identities = set([deserialize(ObjectBase, val)._identity])
else:
fieldId = self._currentTypeMap().lookupOrAdd(
schema_name, typename, field)
identities = set(
self._kvstore.getSetMembers(
IndexId(fieldId=fieldId, indexValue=val)))
return typedef, identities
def _indexValuesToSetAdds(self, indexValues):
# indexValues contains (schema:typename:identity:fieldname -> indexHashVal) which builds
# up the indices we need. We need to transpose to a dictionary ordered by the hash values,
# not the identities
t0 = time.time()
heartbeatInterval = getHeartbeatInterval()
setAdds = {}
for iv in indexValues:
val = indexValues[iv]
if val is not None:
fieldId = iv.fieldId
identity = iv.objId
index_key = IndexId(fieldId=fieldId, indexValue=val)
setAdds.setdefault(index_key, set()).add(identity)
# this could take a long time, so we need to keep heartbeating
if time.time() - t0 > heartbeatInterval:
# note that this needs to be 'sendMessage' which sends immediately,
# not, 'write' which queues the message after this function finishes!
self._channel.sendMessage(
ClientToServer.Heartbeat()
)
t0 = time.time()
return setAdds
def _dropConnectionEntry(self, entry):
identity = entry._identity
fieldId = self._currentTypeMap().fieldIdFor("core", "Connection",
" exists")
exists_key = ObjectFieldId(objId=identity, fieldId=fieldId)
exists_index = IndexId(fieldId=fieldId,
indexValue=indexValueFor(bool, True))
self._handleNewTransaction(None, {exists_key: None}, {},
{exists_index: set([identity])}, [], [],
self._cur_transaction_num)
def _removeOldDeadConnections(self):
fieldId = self._currentTypeMap().fieldIdFor("core", "Connection",
" exists")
exists_index = IndexId(fieldId=fieldId,
indexValue=indexValueFor(bool, True))
oldIds = self._kvstore.getSetMembers(exists_index)
if oldIds:
self._kvstore.setSeveral(
{
ObjectFieldId(objId=identity, fieldId=fieldId): None
for identity in oldIds
}, {}, {exists_index: set(oldIds)})
def _createConnectionEntry(self):
identity = self.identityProducer.createIdentity()
fieldId = self._currentTypeMap().fieldIdFor("core", "Connection",
" exists")
exists_key = ObjectFieldId(objId=identity, fieldId=fieldId)
exists_index = IndexId(fieldId=fieldId,
indexValue=indexValueFor(bool, True))
identityRoot = self.allocateNewIdentityRoot()
self._handleNewTransaction(None, {exists_key: serialize(bool, True)},
{exists_index: set([identity])}, {}, [], [],
self._cur_transaction_num)
return core_schema.Connection.fromIdentity(identity), identityRoot
def _increaseBroadcastTransactionToInclude(self, channel, indexKey, newIds,
key_value, set_adds,
set_removes):
# we need to include all the data for the objects in 'newIds' to the transaction
# that we're broadcasting
fieldId = indexKey.fieldId
fieldDef = self._currentTypeMap().fieldIdToDef[fieldId]
typedef = channel.definedSchemas.get(
fieldDef.schema)[fieldDef.typename]
key_value.update(
self._loadValuesForObject(channel, fieldDef.schema,
fieldDef.typename, newIds))
reverseKeys = []
for index_name in typedef.indices:
for ident in newIds:
reverseKeys.append(
ObjectFieldId(fieldId=fieldId,
objId=ident,
isIndexValue=True))
reverseVals = self._kvstore.getSeveral(reverseKeys)
reverseKVMap = {
reverseKeys[i]: reverseVals[i]
for i in range(len(reverseKeys))
}
for index_name in typedef.indices:
fieldId = self._currentTypeMap().fieldIdFor(
fieldDef.schema, fieldDef.typename, index_name)
for ident in newIds:
fieldval = reverseKVMap.get(
ObjectFieldId(fieldId=fieldId, objId=ident))
if fieldval is not None:
ik = IndexId(fieldId=fieldId, indexValue=fieldval)
set_adds.setdefault(ik, set()).add(ident)
def _markSubscriptionComplete(self, schema, typename, fieldname_and_value,
identities, connectedChannel, isLazy):
if fieldname_and_value is not None:
# this is an index subscription
for ident in identities:
self._id_to_channel.setdefault(ident,
set()).add(connectedChannel)
connectedChannel.subscribedIds.add(ident)
if fieldname_and_value[0] != '_identity':
fieldId = self._currentTypeMap().fieldIdFor(
schema, typename, fieldname_and_value[0])
index_key = IndexId(fieldId=fieldId,
indexValue=fieldname_and_value[1])
if index_key not in self._index_to_channel:
self._index_to_channel[index_key] = set()
self._index_to_channel[index_key].add(connectedChannel)
connectedChannel.subscribedIndexKeys[
index_key] = -1 if not isLazy else self._cur_transaction_num
else:
# an object's identity cannot change, so we don't need to track our subscription to it
assert not isLazy
else:
# this is a type-subscription
for fieldname in connectedChannel.definedSchemas[schema][
typename].fields:
fieldId = self._currentTypeMap().fieldIdFor(
schema, typename, fieldname)
if fieldId not in self._field_id_to_channel:
self._field_id_to_channel[fieldId] = set()
self._field_id_to_channel[fieldId].add(connectedChannel)
connectedChannel.subscribedFields[
fieldId] = -1 if not isLazy else self._cur_transaction_num