def test_can_serialize_type_functions(self):
@TypeFunction
def List(T):
ListT = Forward("ListT")
return ListT.define(
Alternative("List", Node={
"head": T,
"tail": ListT
}, Empty={}))
context = SerializationContext({'List': List})
self.assertIs(context.deserialize(context.serialize(List(int))),
List(int))
self.assertIsInstance(
context.deserialize(context.serialize(List(int).Empty())),
List(int))
list_of_int = List(int)
list_of_list = List(list_of_int)
l0 = list_of_int.Empty()
l_l = list_of_list.Node(head=l0, tail=list_of_list.Empty())
self.assertEqual(context.deserialize(context.serialize(l_l)), l_l)
def test_serialize_lambdas(self):
sc = SerializationContext({})
with tempfile.TemporaryDirectory() as tf:
fpath = os.path.join(tf, "weird_serialization_test.py")
with open(fpath, "w") as f:
f.write("def f(x):\n return x + 1\n")
sys.path.append(tf)
m = importlib.import_module('weird_serialization_test')
#verify we can serialize this
deserialized_f = sc.deserialize(sc.serialize(m.f))
self.assertEqual(deserialized_f(10), 11)
assert not os.path.exists(fpath)
ast_util.clearAllCaches()
#at this point, the backing data for serialization is not there
#and also, the cache is cleared.
deserialized_f_2 = sc.deserialize(sc.serialize(deserialized_f))
self.assertEqual(deserialized_f_2(10), 11)
def test_serialize_named_tuple_subclass(self):
class X(NamedTuple(x=int)):
def f(self):
return self.x
ts = SerializationContext({'X': X})
self.assertIs(ping_pong(X, ts), X)
self.assertTrue(ts.serialize(X(x=20)) != ts.serialize(X(x=21)))
self.check_idempotence(X(x=20), ts)
def test_serialize_and_numpy(self):
x = numpy.ones(10000)
ts = SerializationContext()
self.assertTrue(numpy.all(x == ts.deserialize(ts.serialize(x))))
sizeCompressed = len(ts.serialize(x))
ts.compressionEnabled = False
self.assertTrue(numpy.all(x == ts.deserialize(ts.serialize(x))))
sizeNotCompressed = len(ts.serialize(x))
self.assertTrue(sizeNotCompressed > sizeCompressed * 2, (sizeNotCompressed, sizeCompressed))
def test_serialize_memoizes_tuples(self):
ts = SerializationContext()
l = (1, 2, 3)
for i in range(100):
l = (l, l)
self.assertTrue(len(ts.serialize(l)) < (i + 1) * 100)
def test_serialize_large_numpy_arrays(self):
x = SerializationContext({})
a = numpy.arange(100000000)
a2 = x.deserialize(x.serialize(a))
self.assertTrue(numpy.all(a == a2))
def test_inject_exception_into_context(self):
NT = NamedTuple()
context = SerializationContext({'NT': NT})
context2 = SerializationContext({'NT': NT})
def throws(*args):
raise Exception("Test Exception")
context.nameForObject = throws
context2.objectFromName = throws
with self.assertRaisesRegex(Exception, "Test Exception"):
context.serialize(NT)
data = context2.serialize(NT)
with self.assertRaisesRegex(Exception, "Test Exception"):
context2.deserialize(data)
def test_serialize_dict_doesnt_leak(self):
T = Dict(int, int)
d = T({i: i+1 for i in range(100)})
x = SerializationContext({})
usage = currentMemUsageMb()
for _ in range(20000):
x.deserialize(x.serialize(d))
self.assertLess(currentMemUsageMb(), usage+1)
def test_serialize_dict(self):
x = SerializationContext({})
d = Dict(str, str)()
d["hi"] = "hi"
d["a"] = "a"
d2 = x.deserialize(x.serialize(d))
self.assertEqual(d, d2)
def test_serialize_datetime_objects(self):
x = SerializationContext({})
d = datetime.date.today()
d2 = x.deserialize(x.serialize(d))
self.assertEqual(d, d2, (d, type(d)))
d = datetime.datetime.now()
d2 = x.deserialize(x.serialize(d))
self.assertEqual(d, d2, (d, type(d)))
d = datetime.timedelta(days=1)
d2 = x.deserialize(x.serialize(d))
self.assertEqual(d, d2, (d, type(d)))
d = datetime.datetime.now().time()
d2 = x.deserialize(x.serialize(d))
self.assertEqual(d, d2, (d, type(d)))
d = pytz.timezone("America/New_York")
d2 = x.deserialize(x.serialize(d))
self.assertEqual(d, d2, (d, type(d)))
d = pytz.timezone("America/New_York").localize(datetime.datetime.now())
d2 = x.deserialize(x.serialize(d))
self.assertEqual(d, d2, (d, type(d)))
def test_serialize_recursive_dict_more(self):
D = Forward("D")
D = D.define(Dict(str, OneOf(str, D)))
x = SerializationContext({"D": D})
d = D()
d["hi"] = "bye"
d["recurses"] = d
d2 = x.deserialize(x.serialize(d))
self.assertEqual(d2['recurses']['recurses']['hi'], 'bye')
def test_serialize_large_lists(self):
x = SerializationContext({})
lst = ListOf(ListOf(int))()
lst.resize(100)
for sublist in lst:
sublist.resize(1000000)
t0 = time.time()
l2 = x.deserialize(x.serialize(lst))
print(time.time() - t0, " to roundtrip")
self.assertEqual(lst, l2)
def test_serialize_result_of_decorator(self):
sc = SerializationContext({})
def decorator(f):
def addsOne(x):
return f(x) + 1
return addsOne
@decorator
def g(x):
return x + 1
g2 = sc.deserialize(sc.serialize(g))
self.assertEqual(g2(10), g(10))
def test_serialize_class_instance(self):
class A:
def __init__(self, x):
self.x = x
def f(self):
return b"an embedded string"
ts = SerializationContext({'A': A})
serialization = ts.serialize(A(10))
self.assertTrue(b'an embedded string' not in serialization)
anA = ts.deserialize(serialization)
self.assertEqual(anA.x, 10)
anA2 = deserialize(A, serialize(A, A(10), ts), ts)
self.assertEqual(anA2.x, 10)
