def __init__(self, channel, connectionMetadata=None): self._channel = channel self._transaction_callbacks = {} self._connectionMetadata = connectionMetadata or {} self._lock = threading.RLock() # transaction of what's in the KV store self._cur_transaction_num = 0 self.serializationContext = TypedPythonCodebase.coreSerializationContext().withoutCompression() # a datastructure that keeps track of all the different versions of the objects # we have mapped in. self._connection_state = DatabaseConnectionState() self._connection_state.setSerializationContext(self.serializationContext) self._connection_state.setTriggerLazyLoad(self.loadLazyObject) self._lazy_object_read_blocks = {} self.initialized = threading.Event() self.disconnected = threading.Event() # for each schema name we've sent, an event that's triggered # when the server has acknowledged the schema and given us a definition self._schema_response_events = {} self._fields_to_field_ids = Dict(FieldDefinition, int)() self._field_id_to_schema_and_typename = {} self._field_id_to_field_def = Dict(int, FieldDefinition)() self.connectionObject = None # transaction handlers. These must be nonblocking since we call them under lock self._onTransactionHandlers = set() self._flushEvents = {} # set(schema) self._schemas = set() self._messages_received = 0 self._pendingSubscriptions = {} # from (schema, typename, fieldname_and_val) -> {'values', 'index_values', 'identities'} # where (fieldname_and_val) is OneOf(None, (str, IndexValue)) self._subscription_buildup = {} self._channel.setServerToClientHandler(self._onMessage) self._flushIx = 0 self._largeSubscriptionHeartbeatDelay = 0 self._logger = logging.getLogger(__name__) self._auth_token = None self._max_tid_by_schema = {} self._max_tid_by_schema_and_type = {}
class TypeMap(Class): fieldDefToId = Member(Dict(FieldDefinition, int)) fieldIdToDef = Member(Dict(int, FieldDefinition)) def __len__(self): return len(self.fieldDefToId) def lookupOrAdd(self, schema, typename, fieldname): key = FieldDefinition(schema=schema, typename=typename, fieldname=fieldname) if key not in self.fieldDefToId: fieldId = len(self.fieldDefToId) self.fieldDefToId[key] = fieldId self.fieldIdToDef[fieldId] = key return self.fieldDefToId[key] def fieldIdFor(self, schema, typename, fieldname): key = FieldDefinition(schema=schema, typename=typename, fieldname=fieldname) return self.fieldDefToId.get(key)
def test_serialize_dict_doesnt_leak(self): T = Dict(int, int) d = T({i: i+1 for i in range(100)}) x = SerializationContext({}) usage = currentMemUsageMb() for _ in range(20000): x.deserialize(x.serialize(d)) self.assertLess(currentMemUsageMb(), usage+1)
def test_serialize_dict(self): x = SerializationContext({}) d = Dict(str, str)() d["hi"] = "hi" d["a"] = "a" d2 = x.deserialize(x.serialize(d)) self.assertEqual(d, d2)
def test_dict(self): T = Dict(int, int) self.assertEqual( serialize(T, T({ 1: 2, 33: 44 })), BEGIN_COMPOUND(0) + VARINT(0) + unsignedVarint(0) + # for the id VARINT(0) + unsignedVarint(2) + # for the size VARINT(0) + signedVarint(1) + VARINT(0) + signedVarint(2) + VARINT(0) + signedVarint(33) + VARINT(0) + signedVarint(44) + END_COMPOUND())
def test_serialize_recursive_dict_more(self): D = Forward("D") D = D.define(Dict(str, OneOf(str, D))) x = SerializationContext({"D": D}) d = D() d["hi"] = "bye" d["recurses"] = d d2 = x.deserialize(x.serialize(d)) self.assertEqual(d2['recurses']['recurses']['hi'], 'bye')
def test_recursive_dicts(self): D = Forward("D") D = D.define(Dict(int, OneOf(int, D))) dInst = D() dInst[10] = dInst dInst[20] = 20 self.assertEqual(dInst[10][10][10][20], 20) # stringifying it shouldn't blow up str(dInst) self.assertEqual(dInst, dInst[10])
def instancesOf(T): """Produce some instances of type T""" if T is type(None): # noqa return [None] if T is bool: return [True, False] if T in (int, float, Int8, Int16, Int32, Float32): return [T(x) for x in [-2, -1, 0, 1, 2]] if T in (UInt8, UInt16, UInt32, UInt64): return [T(x) for x in [0, 1, 2]] if T is str: return ['', 'a', 'b', 'ab', 'ba'] if T is bytes: return [b'', b'a', b'b', b'ab', b'ba'] if T is ListOf(int): return [ ListOf(int)(), ListOf(int)([1]), ListOf(int)([2]), ListOf(int)([1, 2]), ListOf(int)([2, 1]) ] if T is ListOf(str): return [ ListOf(str)(), ListOf(str)(['a']), ListOf(str)(['b']), ListOf(str)(['a', 'b']), ListOf(str)(['b', 'a']) ] if T is TupleOf(int): return [ TupleOf(int)(), TupleOf(int)([1]), TupleOf(int)([2]), TupleOf(int)([1, 2]), TupleOf(int)([2, 1]) ] if T is TupleOf(str): return [ TupleOf(str)(), TupleOf(str)(['a']), TupleOf(str)(['b']), TupleOf(str)(['a', 'b']), TupleOf(str)(['b', 'a']) ] if T is Tuple(int, int, int): return [T((1, 2, 3)), T((2, 2, 3)), T((3, 2, 1))] if T is Tuple(str, int, str): return [T(('1', 2, '3')), T(('2', 2, '3')), T(('3', 2, '1'))] if T is Dict(int, int): return [T({1: 2}), T({3: 4}), T({1: 2, 3: 4})] if T is Dict(str, str): return [T({'1': '2'}), T({'3': '4'}), T({'1': '2', '3': '4'})] assert False, f"Can't make instances of {T}"
return tuple if T.__typed_python_category__ == "Dict": return dict assert False, f"No pyType for {T}" # the types we will test. types = [ type(None), bool, int, float, str, bytes, UInt8, UInt16, UInt32, UInt64, Int8, Int16, Int32, Float32, ListOf(int), ListOf(str), TupleOf(int), TupleOf(str), Dict(int, int), Dict(str, str), Tuple(int, int, int), Tuple(str, int, str) ] def instancesOf(T): """Produce some instances of type T""" if T is type(None): # noqa return [None] if T is bool: return [True, False] if T in (int, float, Int8, Int16, Int32, Float32):
def __init__(self, kvstore, auth_token): self._kvstore = kvstore self._auth_token = auth_token self.serializationContext = TypedPythonCodebase.coreSerializationContext( ).withoutCompression() self._lock = threading.RLock() self.verbose = False self._gc_interval = DEFAULT_GC_INTERVAL self._typeMap = None # InMemoryChannel or ServerToClientProtocol -> ConnectedChannel self._clientChannels = {} # id of the next transaction self._cur_transaction_num = 0 # for each key, the last version number we committed self._version_numbers = {} self._version_numbers_timestamps = {} # _field_id to set(subscribed channel) self._field_id_to_channel = {} # index-stringname to set(subscribed channel) self._index_to_channel = Dict(IndexId, object)() # for each individually subscribed ID, a set of channels self._id_to_channel = {} self.longTransactionThreshold = 1.0 self.logFrequency = 10.0 self.MAX_NORMAL_TO_SEND_SYNCHRONOUSLY = 1000 self.MAX_LAZY_TO_SEND_SYNCHRONOUSLY = 10000 self._transactions = 0 self._keys_set = 0 self._index_values_updated = 0 self._subscriptions_written = 0 self._subscriptionResponseThread = None self._shouldStop = threading.Event() # a queue of queue-subscription messages. we have to handle # these on another thread because they can be quite large, and we don't want # to prevent message processing on the main thread. self._subscriptionQueue = queue.Queue() # if we're building a subscription up, all the objects that have changed while our # lock was released. self._pendingSubscriptionRecheck = None # fault injector to test this thing self._subscriptionBackgroundThreadCallback = None self._lazyLoadCallback = None self._last_garbage_collect_timestamp = None self.identityProducer = IdentityProducer( self.allocateNewIdentityRoot()) self._logger = logging.getLogger(__name__) self._removeOldDeadConnections()