def set(self, key, value):
with self.lock:
if value is None:
self.redis.delete(serialize(KeyType, key))
if key in self.cache:
del self.cache[key]
else:
self.redis.set(serialize(KeyType, key), value)
self.cache[key] = value
def test_alternative(self):
A = Alternative("A", X=dict(a=int), Y=dict(b=float, c=float))
self.assertEqual(serialize(A, A.X(a=10)),
SINGLE(0) + SINGLE(0) + VARINT(0) + signedVarint(10))
self.assertEqual(
serialize(A, A.Y(b=10, c=20.2)),
SINGLE(0) + BEGIN_COMPOUND(1) + BITS_64(0) + floatToBits(10) +
BITS_64(1) + floatToBits(20.2) + END_COMPOUND())
def test(s):
utfForm = s.encode("utf8")
self.assertEqual(serialize(str, s),
BYTES(0) + unsignedVarint(len(utfForm)) + utfForm)
self.assertEqual(
deserialize(str, serialize(str,
s)).encode('raw_unicode_escape'),
s.encode('raw_unicode_escape'))
def test_single_values(self):
# each value we produce should consist of a single field number (0)
# encoding a value
# None encoded as wire type of 'EMPTY', and no data
self.assertEqual(serialize(None, None), EMPTY(0))
# integers encoded as a varint
self.assertEqual(serialize(int, 0), VARINT(0) + signedVarint(0))
self.assertEqual(serialize(int, -1), VARINT(0) + signedVarint(-1))
self.assertEqual(serialize(int, 1), VARINT(0) + signedVarint(1))
# floats encoded as a BITS_64
self.assertEqual(serialize(float, 1.5), BITS_64(0) + floatToBits(1.5))
# bools encoded as an unsigned varint
self.assertEqual(serialize(bool, False), VARINT(0) + unsignedVarint(0))
self.assertEqual(serialize(bool, True), VARINT(0) + unsignedVarint(1))
# bytes encoded directly
self.assertEqual(serialize(bytes, b"123"),
BYTES(0) + unsignedVarint(3) + b"123")
# strings encoded as utf-8
self.assertEqual(serialize(str, "123"),
BYTES(0) + unsignedVarint(3) + b"123")
def test_serialize_listof(self):
T = ListOf(int)
aList = T()
aPopulatedList = T([1, 2, 3])
self.assertEqual(aList, deserialize(T, serialize(T, aList)))
self.assertEqual(refcount(deserialize(T, serialize(T, aList))), 1)
self.assertEqual(aPopulatedList, deserialize(T, serialize(T, aPopulatedList)))
self.assertEqual(refcount(deserialize(T, serialize(T, aPopulatedList))), 1)
def test_serialization(self):
ints = TupleOf(int)((1,2,3,4))
self.assertEqual(
len(serialize(TupleOf(int), ints)),
36
)
while len(ints) < 1000000:
ints = ints + ints
t0 = time.time()
self.assertEqual(len(serialize(TupleOf(int), ints)), len(ints) * 8 + 4)
print(time.time() - t0, " for ", len(ints))
def test_embedded_messages(self):
T = NamedTuple(x=TupleOf(int))
T_with_message = NamedTuple(x=EmbeddedMessage)
T_with_two_messages = NamedTuple(x=EmbeddedMessage, y=EmbeddedMessage)
T2 = NamedTuple(x=TupleOf(int), y=TupleOf(int))
t = T(x=(1, 2, 3, 4))
tm = deserialize(T_with_message, serialize(T, t))
tm2 = T_with_two_messages(x=tm.x, y=tm.x)
t2 = deserialize(T2, serialize(T_with_two_messages, tm2))
self.assertEqual(t2.x, t.x)
self.assertEqual(t2.y, t.x)
def test_empty_alternatives(self):
a = Alternative(
"Alt",
A={},
B={}
)
self.assertEqual(a.A(), a.A())
self.assertIsInstance(deserialize(a, serialize(a, a.A())), a.A)
self.assertEqual(a.A(), deserialize(a, serialize(a, a.A())))
self.assertEqual(a.B(), a.B())
self.assertNotEqual(a.A(), a.B())
self.assertNotEqual(a.B(), a.A())
def test_oneof(self):
# tuples are a compound with indices on item numbers
T = OneOf(None, int, float, "HI", TupleOf(int))
self.assertEqual(serialize(T, None), SINGLE(0) + EMPTY(0))
self.assertEqual(serialize(T, 1),
SINGLE(0) + VARINT(1) + signedVarint(1))
self.assertEqual(serialize(T, 1.5),
SINGLE(0) + BITS_64(2) + floatToBits(1.5))
self.assertEqual(serialize(T, "HI"), SINGLE(0) + EMPTY(3))
self.assertEqual(
serialize(T, (1, 2, 123)),
SINGLE(0) + BEGIN_COMPOUND(4) +
(VARINT(0) + unsignedVarint(3) + VARINT(0) + signedVarint(1) +
VARINT(0) + signedVarint(2) + VARINT(0) + signedVarint(123)) +
END_COMPOUND())
def get(self, key):
"""Get the value stored in a value-style key, or None if no key exists.
Throws an exception if the value is a set.
"""
with self.lock:
if key in self.cache:
assert not isinstance(self.cache[key],
set), "item is a set, not a string"
return self.cache[key]
success = False
while not success:
try:
result = self.redis.get(serialize(KeyType, key))
success = True
except redis.exceptions.BusyLoadingError:
self._logger.info("Redis is still loading. Waiting...")
time.sleep(1.0)
if result is None:
return result
assert isinstance(result, bytes)
self.cache[key] = result
return result
def test_serialization_roundtrip(self):
badlen = None
for _ in range(100):
producer = RandomValueProducer()
producer.addEvenly(30, 3)
values = producer.all()
for v in values:
ser = serialize(type(v), v)
v2 = deserialize(type(v), ser)
ser2 = serialize(type(v), v2)
self.assertTrue(type(v2) is type(v))
self.assertEqual(ser,ser2)
self.assertEqual(v, v2)
def test_serialize_named_tuples_with_extra_fields(self):
T1 = NamedTuple(x=int)
T2 = NamedTuple(x=int, y=float, z=str)
self.assertEqual(
deserialize(T2, serialize(T1, T1(x=10))),
T2(x=10, y=0.0, z="")
)
def test_dict_to_oneof(self):
t = ConstDict(str,OneOf("A","B","ABCDEF"))
a = t({'a':'A','b':'ABCDEF'})
self.assertEqual(a['a'], "A")
self.assertEqual(a['b'], "ABCDEF")
self.assertEqual(a, deserialize(t,serialize(t,a)))
def test_tuple_of_one_of_fixed_size(self):
t = TupleOf(OneOf(0,1,2,3,4))
ints = tuple([x % 5 for x in range(1000000)])
typedInts = t(ints)
self.assertEqual(len(serialize(t, typedInts)), len(ints) + 4)
self.assertEqual(tuple(typedInts), ints)
def test_serialize_stream_integers(self):
for someInts in [(1, 2), TupleOf(int)((1, 2)), [1, 2]]:
self.assertEqual(
serializeStream(int, someInts),
b"".join([serialize(int, x) for x in someInts])
)
self.assertEqual(
deserializeStream(int, serializeStream(int, someInts)),
TupleOf(int)(someInts)
)
def test_const_dict(self):
T = ConstDict(int, int)
self.assertEqual(
serialize(T, T({
1: 2,
33: 44
})),
BEGIN_COMPOUND(0) + VARINT(0) + unsignedVarint(2) + # for the size
VARINT(0) + signedVarint(1) + VARINT(0) + signedVarint(2) +
VARINT(0) + signedVarint(33) + VARINT(0) + signedVarint(44) +
END_COMPOUND())
def set(self, key, value):
if not isinstance(key, str) and key.isIndexValue and value is not None:
value = serialize(IndexValue, value, None)
assert isinstance(value, bytes) or value is None, (key, value)
with self.lock:
if value is None:
if key in self.values:
del self.values[key]
else:
self.values[key] = value
def test_dict_containment(self):
for _ in range(100):
producer = RandomValueProducer()
producer.addEvenly(20, 2)
values = producer.all()
for v in values:
if str(type(v))[:17] == "<class 'ConstDict":
v = deserialize(type(v), serialize(type(v), v))
for k in v:
self.assertTrue(k in v)
def test_tuple_of_one_of_multi(self):
t = TupleOf(OneOf(int, bool))
someThings = tuple([100 + x % 5 if x % 17 != 0 else bool(x%19) for x in range(1000000)])
typedThings = t(someThings)
self.assertEqual(
len(serialize(t, typedThings)),
sum(2 if isinstance(t,bool) else 9 for t in someThings) + 4
)
self.assertEqual(tuple(typedThings), someThings)
def setSeveral(self, kvs, setAdds=None, setRemoves=None):
new_sets, dropped_sets = set(), set()
with self.lock:
pipe = self.redis.pipeline()
for key, value in kvs.items():
if value is None:
pipe.delete(serialize(KeyType, key))
else:
pipe.set(serialize(KeyType, key), value)
for key, to_add in (setAdds or {}).items():
if key not in self.cache:
self.getSetMembers(key)
for to_add_val in to_add:
pipe.sadd(serialize(KeyType, key),
serialize(SetValue, to_add_val))
for key, to_remove in (setRemoves or {}).items():
if key not in self.cache:
self.getSetMembers(key)
for to_remove_val in to_remove:
pipe.srem(serialize(KeyType, key),
serialize(SetValue, to_remove_val))
pipe.execute()
# update our cache _after_ executing the pipe
for key, value in kvs.items():
if value is None:
if key in self.cache:
del self.cache[key]
else:
self.cache[key] = value
for key, to_add in (setAdds or {}).items():
if to_add:
if key not in self.cache or not self.cache[key]:
self.cache[key] = set()
new_sets.add(key)
for val in to_add:
self.cache[key].add(val)
for key, to_remove in (setRemoves or {}).items():
if to_remove:
assert self.cache.get(key)
for val in to_remove:
self.cache[key].discard(val)
if not self.cache[key]:
dropped_sets.add(key)
del self.cache[key]
return new_sets, dropped_sets
def test_serialize_doesnt_leak(self):
T = TupleOf(int)
def getMem():
return psutil.Process().memory_info().rss / 1024 ** 2
m0 = getMem()
for passIx in range(100):
for i in range(1000):
t = T(list(range(i)))
deserialize(T, serialize(T,t))
self.assertTrue(getMem() < m0 + 100)
def allocateNewIdentityRoot(self):
with self._lock:
curIdentityRoot = self._kvstore.get("identityRoot")
if curIdentityRoot is None:
curIdentityRoot = 0
else:
curIdentityRoot = deserialize(int, curIdentityRoot)
result = curIdentityRoot
self._kvstore.set("identityRoot",
serialize(int, curIdentityRoot + 1))
return result
def sendMessage(self, msg):
try:
assert isinstance(
msg, self.sendType), "message %s is of type %s != %s" % (
msg, type(msg), self.sendType)
dataToSend = serialize(self.sendType, msg)
dataToSend = longToString(len(dataToSend)) + dataToSend
with self.writelock:
self.transport.write(dataToSend)
except Exception:
self._logger.error("Error in AlgebraicProtocol: %s",
traceback.format_exc())
self.transport.close()
def _createConnectionEntry(self):
identity = self.identityProducer.createIdentity()
fieldId = self._currentTypeMap().fieldIdFor("core", "Connection",
" exists")
exists_key = ObjectFieldId(objId=identity, fieldId=fieldId)
exists_index = IndexId(fieldId=fieldId,
indexValue=indexValueFor(bool, True))
identityRoot = self.allocateNewIdentityRoot()
self._handleNewTransaction(None, {exists_key: serialize(bool, True)},
{exists_index: set([identity])}, {}, [], [],
self._cur_transaction_num)
return core_schema.Connection.fromIdentity(identity), identityRoot
def test_tuples(self):
# tuples are a compound with indices on item numbers
T = TupleOf(int)
self.assertEqual(serialize(T, T(())), EMPTY(0))
# A single element tuple is SINGLE + 0=VARINT + count + 0=VARINT + value
self.assertEqual(
serialize(T, T((1, ))),
BEGIN_COMPOUND(0) + VARINT(0) + unsignedVarint(1) + VARINT(0) +
signedVarint(1) + END_COMPOUND())
self.assertEqual(
serialize(T, T((123, 124))),
BEGIN_COMPOUND(0) +
(VARINT(0) + unsignedVarint(2) + VARINT(0) + signedVarint(123) +
VARINT(0) + signedVarint(124)) + END_COMPOUND())
self.assertEqual(
serialize(T, T((123, 124, 125))),
BEGIN_COMPOUND(0) +
(VARINT(0) + unsignedVarint(3) + VARINT(0) + signedVarint(123) +
VARINT(0) + signedVarint(124) + VARINT(0) + signedVarint(125)) +
END_COMPOUND())
def test_serialize_stream_complex(self):
T = OneOf(None, float, str, int, ListOf(int))
for items in [
(1, 2),
("hi", None, 10, [1, 2, 3, 4]),
()]:
self.assertEqual(
serializeStream(T, items),
b"".join([serialize(T, x) for x in items])
)
self.assertEqual(
deserializeStream(T, serializeStream(T, items)),
TupleOf(T)(items)
)
def test_conversion_of_binary_compatible(self):
class T1(NamedTuple(a=int)):
pass
class T2(NamedTuple(a=int)):
pass
class T1Comp(NamedTuple(d=ConstDict(str, T1))):
pass
class T2Comp(NamedTuple(d=ConstDict(str, T1))):
pass
aT1C = T1Comp(d={'a': T1(a=10)})
self.assertEqual(T2Comp(aT1C).d['a'].a, 10)
self.assertEqual(aT1C, deserialize(T1Comp, serialize(T2Comp, aT1C)))
def test_serialize_classes(self):
class AClass(Class):
x = Member(int)
y = Member(float)
T = Tuple(AClass, AClass)
a = AClass(x=10, y=20.0)
a2, a2_copy = deserialize(T, serialize(T, (a, a)))
self.assertEqual(a2.x, 10)
a2.x = 300
self.assertEqual(a2_copy.x, 300)
a2_copy = None
self.assertEqual(refcount(a2), 1)
def test_recursive_list(self):
L = Forward("L")
L = L.define(ListOf(OneOf(int, L)))
listInst = L()
listInst.append(10)
listInst.append(listInst)
self.assertEqual(
serialize(L, listInst),
BEGIN_COMPOUND(0) + (
VARINT(0) + unsignedVarint(0) + # the ID
VARINT(0) + unsignedVarint(2) + # the size
SINGLE(0) + VARINT(0) + signedVarint(10) + SINGLE(0) +
SINGLE(1) +
( # a compound object encoding the second element of the one-of
VARINT(0) + unsignedVarint(0) # the ID and nothing else
)) + END_COMPOUND())
def test_serialize_class_instance(self):
class A:
def __init__(self, x):
self.x = x
def f(self):
return b"an embedded string"
ts = SerializationContext({'A': A})
serialization = ts.serialize(A(10))
self.assertTrue(b'an embedded string' not in serialization)
anA = ts.deserialize(serialization)
self.assertEqual(anA.x, 10)
anA2 = deserialize(A, serialize(A, A(10), ts), ts)
self.assertEqual(anA2.x, 10)
