def test_serialize_listof(self): T = ListOf(int) aList = T() aPopulatedList = T([1, 2, 3]) self.assertEqual(aList, deserialize(T, serialize(T, aList))) self.assertEqual(refcount(deserialize(T, serialize(T, aList))), 1) self.assertEqual(aPopulatedList, deserialize(T, serialize(T, aPopulatedList))) self.assertEqual(refcount(deserialize(T, serialize(T, aPopulatedList))), 1)
def test_embedded_messages(self): T = NamedTuple(x=TupleOf(int)) T_with_message = NamedTuple(x=EmbeddedMessage) T_with_two_messages = NamedTuple(x=EmbeddedMessage, y=EmbeddedMessage) T2 = NamedTuple(x=TupleOf(int), y=TupleOf(int)) t = T(x=(1, 2, 3, 4)) tm = deserialize(T_with_message, serialize(T, t)) tm2 = T_with_two_messages(x=tm.x, y=tm.x) t2 = deserialize(T2, serialize(T_with_two_messages, tm2)) self.assertEqual(t2.x, t.x) self.assertEqual(t2.y, t.x)
def test_empty_alternatives(self): a = Alternative( "Alt", A={}, B={} ) self.assertEqual(a.A(), a.A()) self.assertIsInstance(deserialize(a, serialize(a, a.A())), a.A) self.assertEqual(a.A(), deserialize(a, serialize(a, a.A()))) self.assertEqual(a.B(), a.B()) self.assertNotEqual(a.A(), a.B()) self.assertNotEqual(a.B(), a.A())
def test_serialize_doesnt_leak(self): T = TupleOf(int) def getMem(): return psutil.Process().memory_info().rss / 1024 ** 2 m0 = getMem() for passIx in range(100): for i in range(1000): t = T(list(range(i))) deserialize(T, serialize(T,t)) self.assertTrue(getMem() < m0 + 100)
def _parseSubscriptionMsg(self, channel, msg): schema_name = msg.schema definition = channel.definedSchemas.get(schema_name) assert definition is not None, "can't subscribe to a schema we don't know about!" assert msg.typename is not None typename = msg.typename assert typename in definition, ( "Can't subscribe to a type we didn't define in the schema: %s not in %s" % (typename, list(definition))) typedef = definition[typename] if msg.fieldname_and_value is None: field, val = " exists", indexValueFor(bool, True) else: field, val = msg.fieldname_and_value if field == '_identity': # single value identities are encoded as integers identities = set([deserialize(ObjectBase, val)._identity]) else: fieldId = self._currentTypeMap().lookupOrAdd( schema_name, typename, field) identities = set( self._kvstore.getSetMembers( IndexId(fieldId=fieldId, indexValue=val))) return typedef, identities
def test_serialize_named_tuples_with_extra_fields(self): T1 = NamedTuple(x=int) T2 = NamedTuple(x=int, y=float, z=str) self.assertEqual( deserialize(T2, serialize(T1, T1(x=10))), T2(x=10, y=0.0, z="") )
def test_dict_to_oneof(self): t = ConstDict(str,OneOf("A","B","ABCDEF")) a = t({'a':'A','b':'ABCDEF'}) self.assertEqual(a['a'], "A") self.assertEqual(a['b'], "ABCDEF") self.assertEqual(a, deserialize(t,serialize(t,a)))
def test(s): utfForm = s.encode("utf8") self.assertEqual(serialize(str, s), BYTES(0) + unsignedVarint(len(utfForm)) + utfForm) self.assertEqual( deserialize(str, serialize(str, s)).encode('raw_unicode_escape'), s.encode('raw_unicode_escape'))
def test_dict_containment(self): for _ in range(100): producer = RandomValueProducer() producer.addEvenly(20, 2) values = producer.all() for v in values: if str(type(v))[:17] == "<class 'ConstDict": v = deserialize(type(v), serialize(type(v), v)) for k in v: self.assertTrue(k in v)
def allocateNewIdentityRoot(self): with self._lock: curIdentityRoot = self._kvstore.get("identityRoot") if curIdentityRoot is None: curIdentityRoot = 0 else: curIdentityRoot = deserialize(int, curIdentityRoot) result = curIdentityRoot self._kvstore.set("identityRoot", serialize(int, curIdentityRoot + 1)) return result
def get(self, key): with self.lock: if key not in self.values: return None val = self.values.get(key) assert isinstance(val, bytes), key if not isinstance(key, str) and key.isIndexValue and val is not None: return deserialize(IndexValue, val, None) return val
def test_serialization_roundtrip(self): badlen = None for _ in range(100): producer = RandomValueProducer() producer.addEvenly(30, 3) values = producer.all() for v in values: ser = serialize(type(v), v) v2 = deserialize(type(v), ser) ser2 = serialize(type(v), v2) self.assertTrue(type(v2) is type(v)) self.assertEqual(ser,ser2) self.assertEqual(v, v2)
def test_serialize_classes(self): class AClass(Class): x = Member(int) y = Member(float) T = Tuple(AClass, AClass) a = AClass(x=10, y=20.0) a2, a2_copy = deserialize(T, serialize(T, (a, a))) self.assertEqual(a2.x, 10) a2.x = 300 self.assertEqual(a2_copy.x, 300) a2_copy = None self.assertEqual(refcount(a2), 1)
def test_conversion_of_binary_compatible(self): class T1(NamedTuple(a=int)): pass class T2(NamedTuple(a=int)): pass class T1Comp(NamedTuple(d=ConstDict(str, T1))): pass class T2Comp(NamedTuple(d=ConstDict(str, T1))): pass aT1C = T1Comp(d={'a': T1(a=10)}) self.assertEqual(T2Comp(aT1C).d['a'].a, 10) self.assertEqual(aT1C, deserialize(T1Comp, serialize(T2Comp, aT1C)))
def test_serialize_class_instance(self): class A: def __init__(self, x): self.x = x def f(self): return b"an embedded string" ts = SerializationContext({'A': A}) serialization = ts.serialize(A(10)) self.assertTrue(b'an embedded string' not in serialization) anA = ts.deserialize(serialization) self.assertEqual(anA.x, 10) anA2 = deserialize(A, serialize(A, A(10), ts), ts) self.assertEqual(anA2.x, 10)
def unwrapSerializedDatabaseValue(serializationContext, dbValWithPyrep, field_type): assert field_type is not None if dbValWithPyrep is None: return default_initialize(field_type) if isinstance(dbValWithPyrep, str): dbValWithPyrep = SerializedDatabaseValue( bytes.fromhex(dbValWithPyrep), {}) if dbValWithPyrep.serializedByteRep is None: return default_initialize(field_type) if dbValWithPyrep.pyRep.get(serializationContext) is None: dbValWithPyrep.pyRep[serializationContext] = deserialize( field_type, dbValWithPyrep.serializedByteRep, serializationContext) return dbValWithPyrep.pyRep[serializationContext]
def data_received(self, data): assert isinstance(data, bytes) self.buffer += data while len(self.buffer) >= longLength: bytesToRead = stringToLong(self.buffer[:longLength]) if bytesToRead + longLength <= len(self.buffer): toConsume = self.buffer[longLength:bytesToRead + longLength] self.buffer = self.buffer[bytesToRead + longLength:] if toConsume: try: self.messageReceived( deserialize(self.receiveType, bytes(toConsume))) except Exception: self._logger.error("Error in AlgebraicProtocol: %s", traceback.format_exc()) self.transport.close() else: return
def test_const_dict(self): t = ConstDict(str,str) self.assertEqual(len(t()), 0) self.assertEqual(len(t({})), 0) self.assertEqual(len(t({'a':'b'})), 1) self.assertEqual(t({'a':'b'})['a'], 'b') self.assertEqual(t({'a':'b','b':'c'})['b'], 'c') self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b'})))) self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b','b':'c'})))) self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d'})))) self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d','d':'e'})))) self.assertTrue("c" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d','def':'e'})))) self.assertTrue("def" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d','def':'e'}))))
def getSetMembers(self, key): with self.lock: if key in self.cache: assert isinstance(self.cache[key], set), "item is a string, not a set" return self.cache[key] success = False while not success: try: vals = self.redis.smembers(serialize(KeyType, key)) success = True except redis.exceptions.BusyLoadingError: self._logger.info("Redis is still loading. Waiting...") time.sleep(1.0) if vals: # set members are encoded as bytes but must be a 'SetValue' self.cache[key] = set([deserialize(SetValue, k) for k in vals]) return self.cache[key] else: return set()
def getSeveral(self, keys): """Get the values (or None) stored in several value-style keys.""" with self.lock: needed_keys = [ serialize(KeyType, k) for k in keys if k not in self.cache ] if needed_keys: success = False while not success: try: vals = self.redis.mget(needed_keys) success = True except redis.exceptions.BusyLoadingError: self._logger.info("Redis is still loading. Waiting...") time.sleep(1.0) for ix in range(len(needed_keys)): if vals[ix] is not None: self.cache[deserialize(KeyType, needed_keys[ix])] = vals[ix] return [self.cache.get(k, None) for k in keys]
def test_deserialize_primitive(self): x = deserialize(str, serialize(str, "a")) self.assertTrue(isinstance(x,str))