def test_serialize_listof(self):
T = ListOf(int)
aList = T()
aPopulatedList = T([1, 2, 3])
self.assertEqual(aList, deserialize(T, serialize(T, aList)))
self.assertEqual(refcount(deserialize(T, serialize(T, aList))), 1)
self.assertEqual(aPopulatedList, deserialize(T, serialize(T, aPopulatedList)))
self.assertEqual(refcount(deserialize(T, serialize(T, aPopulatedList))), 1)
def test_embedded_messages(self):
T = NamedTuple(x=TupleOf(int))
T_with_message = NamedTuple(x=EmbeddedMessage)
T_with_two_messages = NamedTuple(x=EmbeddedMessage, y=EmbeddedMessage)
T2 = NamedTuple(x=TupleOf(int), y=TupleOf(int))
t = T(x=(1, 2, 3, 4))
tm = deserialize(T_with_message, serialize(T, t))
tm2 = T_with_two_messages(x=tm.x, y=tm.x)
t2 = deserialize(T2, serialize(T_with_two_messages, tm2))
self.assertEqual(t2.x, t.x)
self.assertEqual(t2.y, t.x)
def test_empty_alternatives(self):
a = Alternative(
"Alt",
A={},
B={}
)
self.assertEqual(a.A(), a.A())
self.assertIsInstance(deserialize(a, serialize(a, a.A())), a.A)
self.assertEqual(a.A(), deserialize(a, serialize(a, a.A())))
self.assertEqual(a.B(), a.B())
self.assertNotEqual(a.A(), a.B())
self.assertNotEqual(a.B(), a.A())
def test_serialize_doesnt_leak(self):
T = TupleOf(int)
def getMem():
return psutil.Process().memory_info().rss / 1024 ** 2
m0 = getMem()
for passIx in range(100):
for i in range(1000):
t = T(list(range(i)))
deserialize(T, serialize(T,t))
self.assertTrue(getMem() < m0 + 100)
def _parseSubscriptionMsg(self, channel, msg):
schema_name = msg.schema
definition = channel.definedSchemas.get(schema_name)
assert definition is not None, "can't subscribe to a schema we don't know about!"
assert msg.typename is not None
typename = msg.typename
assert typename in definition, (
"Can't subscribe to a type we didn't define in the schema: %s not in %s"
% (typename, list(definition)))
typedef = definition[typename]
if msg.fieldname_and_value is None:
field, val = " exists", indexValueFor(bool, True)
else:
field, val = msg.fieldname_and_value
if field == '_identity':
# single value identities are encoded as integers
identities = set([deserialize(ObjectBase, val)._identity])
else:
fieldId = self._currentTypeMap().lookupOrAdd(
schema_name, typename, field)
identities = set(
self._kvstore.getSetMembers(
IndexId(fieldId=fieldId, indexValue=val)))
return typedef, identities
def test_serialize_named_tuples_with_extra_fields(self):
T1 = NamedTuple(x=int)
T2 = NamedTuple(x=int, y=float, z=str)
self.assertEqual(
deserialize(T2, serialize(T1, T1(x=10))),
T2(x=10, y=0.0, z="")
)
def test_dict_to_oneof(self):
t = ConstDict(str,OneOf("A","B","ABCDEF"))
a = t({'a':'A','b':'ABCDEF'})
self.assertEqual(a['a'], "A")
self.assertEqual(a['b'], "ABCDEF")
self.assertEqual(a, deserialize(t,serialize(t,a)))
def test(s):
utfForm = s.encode("utf8")
self.assertEqual(serialize(str, s),
BYTES(0) + unsignedVarint(len(utfForm)) + utfForm)
self.assertEqual(
deserialize(str, serialize(str,
s)).encode('raw_unicode_escape'),
s.encode('raw_unicode_escape'))
def test_dict_containment(self):
for _ in range(100):
producer = RandomValueProducer()
producer.addEvenly(20, 2)
values = producer.all()
for v in values:
if str(type(v))[:17] == "<class 'ConstDict":
v = deserialize(type(v), serialize(type(v), v))
for k in v:
self.assertTrue(k in v)
def allocateNewIdentityRoot(self):
with self._lock:
curIdentityRoot = self._kvstore.get("identityRoot")
if curIdentityRoot is None:
curIdentityRoot = 0
else:
curIdentityRoot = deserialize(int, curIdentityRoot)
result = curIdentityRoot
self._kvstore.set("identityRoot",
serialize(int, curIdentityRoot + 1))
return result
def get(self, key):
with self.lock:
if key not in self.values:
return None
val = self.values.get(key)
assert isinstance(val, bytes), key
if not isinstance(key,
str) and key.isIndexValue and val is not None:
return deserialize(IndexValue, val, None)
return val
def test_serialization_roundtrip(self):
badlen = None
for _ in range(100):
producer = RandomValueProducer()
producer.addEvenly(30, 3)
values = producer.all()
for v in values:
ser = serialize(type(v), v)
v2 = deserialize(type(v), ser)
ser2 = serialize(type(v), v2)
self.assertTrue(type(v2) is type(v))
self.assertEqual(ser,ser2)
self.assertEqual(v, v2)
def test_serialize_classes(self):
class AClass(Class):
x = Member(int)
y = Member(float)
T = Tuple(AClass, AClass)
a = AClass(x=10, y=20.0)
a2, a2_copy = deserialize(T, serialize(T, (a, a)))
self.assertEqual(a2.x, 10)
a2.x = 300
self.assertEqual(a2_copy.x, 300)
a2_copy = None
self.assertEqual(refcount(a2), 1)
def test_conversion_of_binary_compatible(self):
class T1(NamedTuple(a=int)):
pass
class T2(NamedTuple(a=int)):
pass
class T1Comp(NamedTuple(d=ConstDict(str, T1))):
pass
class T2Comp(NamedTuple(d=ConstDict(str, T1))):
pass
aT1C = T1Comp(d={'a': T1(a=10)})
self.assertEqual(T2Comp(aT1C).d['a'].a, 10)
self.assertEqual(aT1C, deserialize(T1Comp, serialize(T2Comp, aT1C)))
def test_serialize_class_instance(self):
class A:
def __init__(self, x):
self.x = x
def f(self):
return b"an embedded string"
ts = SerializationContext({'A': A})
serialization = ts.serialize(A(10))
self.assertTrue(b'an embedded string' not in serialization)
anA = ts.deserialize(serialization)
self.assertEqual(anA.x, 10)
anA2 = deserialize(A, serialize(A, A(10), ts), ts)
self.assertEqual(anA2.x, 10)
def unwrapSerializedDatabaseValue(serializationContext, dbValWithPyrep,
field_type):
assert field_type is not None
if dbValWithPyrep is None:
return default_initialize(field_type)
if isinstance(dbValWithPyrep, str):
dbValWithPyrep = SerializedDatabaseValue(
bytes.fromhex(dbValWithPyrep), {})
if dbValWithPyrep.serializedByteRep is None:
return default_initialize(field_type)
if dbValWithPyrep.pyRep.get(serializationContext) is None:
dbValWithPyrep.pyRep[serializationContext] = deserialize(
field_type, dbValWithPyrep.serializedByteRep,
serializationContext)
return dbValWithPyrep.pyRep[serializationContext]
def data_received(self, data):
assert isinstance(data, bytes)
self.buffer += data
while len(self.buffer) >= longLength:
bytesToRead = stringToLong(self.buffer[:longLength])
if bytesToRead + longLength <= len(self.buffer):
toConsume = self.buffer[longLength:bytesToRead + longLength]
self.buffer = self.buffer[bytesToRead + longLength:]
if toConsume:
try:
self.messageReceived(
deserialize(self.receiveType, bytes(toConsume)))
except Exception:
self._logger.error("Error in AlgebraicProtocol: %s",
traceback.format_exc())
self.transport.close()
else:
return
def test_const_dict(self):
t = ConstDict(str,str)
self.assertEqual(len(t()), 0)
self.assertEqual(len(t({})), 0)
self.assertEqual(len(t({'a':'b'})), 1)
self.assertEqual(t({'a':'b'})['a'], 'b')
self.assertEqual(t({'a':'b','b':'c'})['b'], 'c')
self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b'}))))
self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b','b':'c'}))))
self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d'}))))
self.assertTrue("a" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d','d':'e'}))))
self.assertTrue("c" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d','def':'e'}))))
self.assertTrue("def" in deserialize(t,serialize(t, t({'a':'b','b':'c','c':'d','def':'e'}))))
def getSetMembers(self, key):
with self.lock:
if key in self.cache:
assert isinstance(self.cache[key],
set), "item is a string, not a set"
return self.cache[key]
success = False
while not success:
try:
vals = self.redis.smembers(serialize(KeyType, key))
success = True
except redis.exceptions.BusyLoadingError:
self._logger.info("Redis is still loading. Waiting...")
time.sleep(1.0)
if vals:
# set members are encoded as bytes but must be a 'SetValue'
self.cache[key] = set([deserialize(SetValue, k) for k in vals])
return self.cache[key]
else:
return set()
def getSeveral(self, keys):
"""Get the values (or None) stored in several value-style keys."""
with self.lock:
needed_keys = [
serialize(KeyType, k) for k in keys if k not in self.cache
]
if needed_keys:
success = False
while not success:
try:
vals = self.redis.mget(needed_keys)
success = True
except redis.exceptions.BusyLoadingError:
self._logger.info("Redis is still loading. Waiting...")
time.sleep(1.0)
for ix in range(len(needed_keys)):
if vals[ix] is not None:
self.cache[deserialize(KeyType,
needed_keys[ix])] = vals[ix]
return [self.cache.get(k, None) for k in keys]
def test_deserialize_primitive(self):
x = deserialize(str, serialize(str, "a"))
self.assertTrue(isinstance(x,str))
