def __init__(self, channel, connectionMetadata=None):
self._channel = channel
self._transaction_callbacks = {}
self._connectionMetadata = connectionMetadata or {}
self._lock = threading.RLock()
# transaction of what's in the KV store
self._cur_transaction_num = 0
self.serializationContext = TypedPythonCodebase.coreSerializationContext().withoutCompression()
# a datastructure that keeps track of all the different versions of the objects
# we have mapped in.
self._connection_state = DatabaseConnectionState()
self._connection_state.setSerializationContext(self.serializationContext)
self._connection_state.setTriggerLazyLoad(self.loadLazyObject)
self._lazy_object_read_blocks = {}
self.initialized = threading.Event()
self.disconnected = threading.Event()
# for each schema name we've sent, an event that's triggered
# when the server has acknowledged the schema and given us a definition
self._schema_response_events = {}
self._fields_to_field_ids = Dict(FieldDefinition, int)()
self._field_id_to_schema_and_typename = {}
self._field_id_to_field_def = Dict(int, FieldDefinition)()
self.connectionObject = None
# transaction handlers. These must be nonblocking since we call them under lock
self._onTransactionHandlers = set()
self._flushEvents = {}
# set(schema)
self._schemas = set()
self._messages_received = 0
self._pendingSubscriptions = {}
# from (schema, typename, fieldname_and_val) -> {'values', 'index_values', 'identities'}
# where (fieldname_and_val) is OneOf(None, (str, IndexValue))
self._subscription_buildup = {}
self._channel.setServerToClientHandler(self._onMessage)
self._flushIx = 0
self._largeSubscriptionHeartbeatDelay = 0
self._logger = logging.getLogger(__name__)
self._auth_token = None
self._max_tid_by_schema = {}
self._max_tid_by_schema_and_type = {}
class TypeMap(Class):
fieldDefToId = Member(Dict(FieldDefinition, int))
fieldIdToDef = Member(Dict(int, FieldDefinition))
def __len__(self):
return len(self.fieldDefToId)
def lookupOrAdd(self, schema, typename, fieldname):
key = FieldDefinition(schema=schema,
typename=typename,
fieldname=fieldname)
if key not in self.fieldDefToId:
fieldId = len(self.fieldDefToId)
self.fieldDefToId[key] = fieldId
self.fieldIdToDef[fieldId] = key
return self.fieldDefToId[key]
def fieldIdFor(self, schema, typename, fieldname):
key = FieldDefinition(schema=schema,
typename=typename,
fieldname=fieldname)
return self.fieldDefToId.get(key)
def test_serialize_dict_doesnt_leak(self):
T = Dict(int, int)
d = T({i: i+1 for i in range(100)})
x = SerializationContext({})
usage = currentMemUsageMb()
for _ in range(20000):
x.deserialize(x.serialize(d))
self.assertLess(currentMemUsageMb(), usage+1)
def test_serialize_dict(self):
x = SerializationContext({})
d = Dict(str, str)()
d["hi"] = "hi"
d["a"] = "a"
d2 = x.deserialize(x.serialize(d))
self.assertEqual(d, d2)
def test_dict(self):
T = Dict(int, int)
self.assertEqual(
serialize(T, T({
1: 2,
33: 44
})),
BEGIN_COMPOUND(0) + VARINT(0) + unsignedVarint(0) + # for the id
VARINT(0) + unsignedVarint(2) + # for the size
VARINT(0) + signedVarint(1) + VARINT(0) + signedVarint(2) +
VARINT(0) + signedVarint(33) + VARINT(0) + signedVarint(44) +
END_COMPOUND())
def test_serialize_recursive_dict_more(self):
D = Forward("D")
D = D.define(Dict(str, OneOf(str, D)))
x = SerializationContext({"D": D})
d = D()
d["hi"] = "bye"
d["recurses"] = d
d2 = x.deserialize(x.serialize(d))
self.assertEqual(d2['recurses']['recurses']['hi'], 'bye')
def test_recursive_dicts(self):
D = Forward("D")
D = D.define(Dict(int, OneOf(int, D)))
dInst = D()
dInst[10] = dInst
dInst[20] = 20
self.assertEqual(dInst[10][10][10][20], 20)
# stringifying it shouldn't blow up
str(dInst)
self.assertEqual(dInst, dInst[10])
def instancesOf(T):
"""Produce some instances of type T"""
if T is type(None): # noqa
return [None]
if T is bool:
return [True, False]
if T in (int, float, Int8, Int16, Int32, Float32):
return [T(x) for x in [-2, -1, 0, 1, 2]]
if T in (UInt8, UInt16, UInt32, UInt64):
return [T(x) for x in [0, 1, 2]]
if T is str:
return ['', 'a', 'b', 'ab', 'ba']
if T is bytes:
return [b'', b'a', b'b', b'ab', b'ba']
if T is ListOf(int):
return [
ListOf(int)(),
ListOf(int)([1]),
ListOf(int)([2]),
ListOf(int)([1, 2]),
ListOf(int)([2, 1])
]
if T is ListOf(str):
return [
ListOf(str)(),
ListOf(str)(['a']),
ListOf(str)(['b']),
ListOf(str)(['a', 'b']),
ListOf(str)(['b', 'a'])
]
if T is TupleOf(int):
return [
TupleOf(int)(),
TupleOf(int)([1]),
TupleOf(int)([2]),
TupleOf(int)([1, 2]),
TupleOf(int)([2, 1])
]
if T is TupleOf(str):
return [
TupleOf(str)(),
TupleOf(str)(['a']),
TupleOf(str)(['b']),
TupleOf(str)(['a', 'b']),
TupleOf(str)(['b', 'a'])
]
if T is Tuple(int, int, int):
return [T((1, 2, 3)), T((2, 2, 3)), T((3, 2, 1))]
if T is Tuple(str, int, str):
return [T(('1', 2, '3')), T(('2', 2, '3')), T(('3', 2, '1'))]
if T is Dict(int, int):
return [T({1: 2}), T({3: 4}), T({1: 2, 3: 4})]
if T is Dict(str, str):
return [T({'1': '2'}), T({'3': '4'}), T({'1': '2', '3': '4'})]
assert False, f"Can't make instances of {T}"
return tuple
if T.__typed_python_category__ == "Dict":
return dict
assert False, f"No pyType for {T}"
# the types we will test.
types = [
type(None), bool, int, float, str, bytes, UInt8, UInt16, UInt32, UInt64,
Int8, Int16, Int32, Float32,
ListOf(int),
ListOf(str),
TupleOf(int),
TupleOf(str),
Dict(int, int),
Dict(str, str),
Tuple(int, int, int),
Tuple(str, int, str)
]
def instancesOf(T):
"""Produce some instances of type T"""
if T is type(None): # noqa
return [None]
if T is bool:
return [True, False]
if T in (int, float, Int8, Int16, Int32, Float32):
def __init__(self, kvstore, auth_token):
self._kvstore = kvstore
self._auth_token = auth_token
self.serializationContext = TypedPythonCodebase.coreSerializationContext(
).withoutCompression()
self._lock = threading.RLock()
self.verbose = False
self._gc_interval = DEFAULT_GC_INTERVAL
self._typeMap = None
# InMemoryChannel or ServerToClientProtocol -> ConnectedChannel
self._clientChannels = {}
# id of the next transaction
self._cur_transaction_num = 0
# for each key, the last version number we committed
self._version_numbers = {}
self._version_numbers_timestamps = {}
# _field_id to set(subscribed channel)
self._field_id_to_channel = {}
# index-stringname to set(subscribed channel)
self._index_to_channel = Dict(IndexId, object)()
# for each individually subscribed ID, a set of channels
self._id_to_channel = {}
self.longTransactionThreshold = 1.0
self.logFrequency = 10.0
self.MAX_NORMAL_TO_SEND_SYNCHRONOUSLY = 1000
self.MAX_LAZY_TO_SEND_SYNCHRONOUSLY = 10000
self._transactions = 0
self._keys_set = 0
self._index_values_updated = 0
self._subscriptions_written = 0
self._subscriptionResponseThread = None
self._shouldStop = threading.Event()
# a queue of queue-subscription messages. we have to handle
# these on another thread because they can be quite large, and we don't want
# to prevent message processing on the main thread.
self._subscriptionQueue = queue.Queue()
# if we're building a subscription up, all the objects that have changed while our
# lock was released.
self._pendingSubscriptionRecheck = None
# fault injector to test this thing
self._subscriptionBackgroundThreadCallback = None
self._lazyLoadCallback = None
self._last_garbage_collect_timestamp = None
self.identityProducer = IdentityProducer(
self.allocateNewIdentityRoot())
self._logger = logging.getLogger(__name__)
self._removeOldDeadConnections()