def set(self, key, value): with self.lock: if value is None: self.redis.delete(serialize(KeyType, key)) if key in self.cache: del self.cache[key] else: self.redis.set(serialize(KeyType, key), value) self.cache[key] = value
def test_alternative(self): A = Alternative("A", X=dict(a=int), Y=dict(b=float, c=float)) self.assertEqual(serialize(A, A.X(a=10)), SINGLE(0) + SINGLE(0) + VARINT(0) + signedVarint(10)) self.assertEqual( serialize(A, A.Y(b=10, c=20.2)), SINGLE(0) + BEGIN_COMPOUND(1) + BITS_64(0) + floatToBits(10) + BITS_64(1) + floatToBits(20.2) + END_COMPOUND())
def test(s): utfForm = s.encode("utf8") self.assertEqual(serialize(str, s), BYTES(0) + unsignedVarint(len(utfForm)) + utfForm) self.assertEqual( deserialize(str, serialize(str, s)).encode('raw_unicode_escape'), s.encode('raw_unicode_escape'))
def test_single_values(self): # each value we produce should consist of a single field number (0) # encoding a value # None encoded as wire type of 'EMPTY', and no data self.assertEqual(serialize(None, None), EMPTY(0)) # integers encoded as a varint self.assertEqual(serialize(int, 0), VARINT(0) + signedVarint(0)) self.assertEqual(serialize(int, -1), VARINT(0) + signedVarint(-1)) self.assertEqual(serialize(int, 1), VARINT(0) + signedVarint(1)) # floats encoded as a BITS_64 self.assertEqual(serialize(float, 1.5), BITS_64(0) + floatToBits(1.5)) # bools encoded as an unsigned varint self.assertEqual(serialize(bool, False), VARINT(0) + unsignedVarint(0)) self.assertEqual(serialize(bool, True), VARINT(0) + unsignedVarint(1)) # bytes encoded directly self.assertEqual(serialize(bytes, b"123"), BYTES(0) + unsignedVarint(3) + b"123") # strings encoded as utf-8 self.assertEqual(serialize(str, "123"), BYTES(0) + unsignedVarint(3) + b"123")
def test_serialize_listof(self): T = ListOf(int) aList = T() aPopulatedList = T([1, 2, 3]) self.assertEqual(aList, deserialize(T, serialize(T, aList))) self.assertEqual(refcount(deserialize(T, serialize(T, aList))), 1) self.assertEqual(aPopulatedList, deserialize(T, serialize(T, aPopulatedList))) self.assertEqual(refcount(deserialize(T, serialize(T, aPopulatedList))), 1)
def test_serialization(self): ints = TupleOf(int)((1,2,3,4)) self.assertEqual( len(serialize(TupleOf(int), ints)), 36 ) while len(ints) < 1000000: ints = ints + ints t0 = time.time() self.assertEqual(len(serialize(TupleOf(int), ints)), len(ints) * 8 + 4) print(time.time() - t0, " for ", len(ints))
def test_embedded_messages(self): T = NamedTuple(x=TupleOf(int)) T_with_message = NamedTuple(x=EmbeddedMessage) T_with_two_messages = NamedTuple(x=EmbeddedMessage, y=EmbeddedMessage) T2 = NamedTuple(x=TupleOf(int), y=TupleOf(int)) t = T(x=(1, 2, 3, 4)) tm = deserialize(T_with_message, serialize(T, t)) tm2 = T_with_two_messages(x=tm.x, y=tm.x) t2 = deserialize(T2, serialize(T_with_two_messages, tm2)) self.assertEqual(t2.x, t.x) self.assertEqual(t2.y, t.x)
def test_empty_alternatives(self): a = Alternative( "Alt", A={}, B={} ) self.assertEqual(a.A(), a.A()) self.assertIsInstance(deserialize(a, serialize(a, a.A())), a.A) self.assertEqual(a.A(), deserialize(a, serialize(a, a.A()))) self.assertEqual(a.B(), a.B()) self.assertNotEqual(a.A(), a.B()) self.assertNotEqual(a.B(), a.A())
def test_oneof(self): # tuples are a compound with indices on item numbers T = OneOf(None, int, float, "HI", TupleOf(int)) self.assertEqual(serialize(T, None), SINGLE(0) + EMPTY(0)) self.assertEqual(serialize(T, 1), SINGLE(0) + VARINT(1) + signedVarint(1)) self.assertEqual(serialize(T, 1.5), SINGLE(0) + BITS_64(2) + floatToBits(1.5)) self.assertEqual(serialize(T, "HI"), SINGLE(0) + EMPTY(3)) self.assertEqual( serialize(T, (1, 2, 123)), SINGLE(0) + BEGIN_COMPOUND(4) + (VARINT(0) + unsignedVarint(3) + VARINT(0) + signedVarint(1) + VARINT(0) + signedVarint(2) + VARINT(0) + signedVarint(123)) + END_COMPOUND())
def get(self, key): """Get the value stored in a value-style key, or None if no key exists. Throws an exception if the value is a set. """ with self.lock: if key in self.cache: assert not isinstance(self.cache[key], set), "item is a set, not a string" return self.cache[key] success = False while not success: try: result = self.redis.get(serialize(KeyType, key)) success = True except redis.exceptions.BusyLoadingError: self._logger.info("Redis is still loading. Waiting...") time.sleep(1.0) if result is None: return result assert isinstance(result, bytes) self.cache[key] = result return result
def test_serialization_roundtrip(self): badlen = None for _ in range(100): producer = RandomValueProducer() producer.addEvenly(30, 3) values = producer.all() for v in values: ser = serialize(type(v), v) v2 = deserialize(type(v), ser) ser2 = serialize(type(v), v2) self.assertTrue(type(v2) is type(v)) self.assertEqual(ser,ser2) self.assertEqual(v, v2)
def test_serialize_named_tuples_with_extra_fields(self): T1 = NamedTuple(x=int) T2 = NamedTuple(x=int, y=float, z=str) self.assertEqual( deserialize(T2, serialize(T1, T1(x=10))), T2(x=10, y=0.0, z="") )
def test_dict_to_oneof(self): t = ConstDict(str,OneOf("A","B","ABCDEF")) a = t({'a':'A','b':'ABCDEF'}) self.assertEqual(a['a'], "A") self.assertEqual(a['b'], "ABCDEF") self.assertEqual(a, deserialize(t,serialize(t,a)))
def test_tuple_of_one_of_fixed_size(self): t = TupleOf(OneOf(0,1,2,3,4)) ints = tuple([x % 5 for x in range(1000000)]) typedInts = t(ints) self.assertEqual(len(serialize(t, typedInts)), len(ints) + 4) self.assertEqual(tuple(typedInts), ints)
def test_serialize_stream_integers(self): for someInts in [(1, 2), TupleOf(int)((1, 2)), [1, 2]]: self.assertEqual( serializeStream(int, someInts), b"".join([serialize(int, x) for x in someInts]) ) self.assertEqual( deserializeStream(int, serializeStream(int, someInts)), TupleOf(int)(someInts) )
def test_const_dict(self): T = ConstDict(int, int) self.assertEqual( serialize(T, T({ 1: 2, 33: 44 })), BEGIN_COMPOUND(0) + VARINT(0) + unsignedVarint(2) + # for the size VARINT(0) + signedVarint(1) + VARINT(0) + signedVarint(2) + VARINT(0) + signedVarint(33) + VARINT(0) + signedVarint(44) + END_COMPOUND())
def set(self, key, value): if not isinstance(key, str) and key.isIndexValue and value is not None: value = serialize(IndexValue, value, None) assert isinstance(value, bytes) or value is None, (key, value) with self.lock: if value is None: if key in self.values: del self.values[key] else: self.values[key] = value
def test_dict_containment(self): for _ in range(100): producer = RandomValueProducer() producer.addEvenly(20, 2) values = producer.all() for v in values: if str(type(v))[:17] == "<class 'ConstDict": v = deserialize(type(v), serialize(type(v), v)) for k in v: self.assertTrue(k in v)
def test_tuple_of_one_of_multi(self): t = TupleOf(OneOf(int, bool)) someThings = tuple([100 + x % 5 if x % 17 != 0 else bool(x%19) for x in range(1000000)]) typedThings = t(someThings) self.assertEqual( len(serialize(t, typedThings)), sum(2 if isinstance(t,bool) else 9 for t in someThings) + 4 ) self.assertEqual(tuple(typedThings), someThings)
def setSeveral(self, kvs, setAdds=None, setRemoves=None): new_sets, dropped_sets = set(), set() with self.lock: pipe = self.redis.pipeline() for key, value in kvs.items(): if value is None: pipe.delete(serialize(KeyType, key)) else: pipe.set(serialize(KeyType, key), value) for key, to_add in (setAdds or {}).items(): if key not in self.cache: self.getSetMembers(key) for to_add_val in to_add: pipe.sadd(serialize(KeyType, key), serialize(SetValue, to_add_val)) for key, to_remove in (setRemoves or {}).items(): if key not in self.cache: self.getSetMembers(key) for to_remove_val in to_remove: pipe.srem(serialize(KeyType, key), serialize(SetValue, to_remove_val)) pipe.execute() # update our cache _after_ executing the pipe for key, value in kvs.items(): if value is None: if key in self.cache: del self.cache[key] else: self.cache[key] = value for key, to_add in (setAdds or {}).items(): if to_add: if key not in self.cache or not self.cache[key]: self.cache[key] = set() new_sets.add(key) for val in to_add: self.cache[key].add(val) for key, to_remove in (setRemoves or {}).items(): if to_remove: assert self.cache.get(key) for val in to_remove: self.cache[key].discard(val) if not self.cache[key]: dropped_sets.add(key) del self.cache[key] return new_sets, dropped_sets
def test_serialize_doesnt_leak(self): T = TupleOf(int) def getMem(): return psutil.Process().memory_info().rss / 1024 ** 2 m0 = getMem() for passIx in range(100): for i in range(1000): t = T(list(range(i))) deserialize(T, serialize(T,t)) self.assertTrue(getMem() < m0 + 100)
def allocateNewIdentityRoot(self): with self._lock: curIdentityRoot = self._kvstore.get("identityRoot") if curIdentityRoot is None: curIdentityRoot = 0 else: curIdentityRoot = deserialize(int, curIdentityRoot) result = curIdentityRoot self._kvstore.set("identityRoot", serialize(int, curIdentityRoot + 1)) return result
def sendMessage(self, msg): try: assert isinstance( msg, self.sendType), "message %s is of type %s != %s" % ( msg, type(msg), self.sendType) dataToSend = serialize(self.sendType, msg) dataToSend = longToString(len(dataToSend)) + dataToSend with self.writelock: self.transport.write(dataToSend) except Exception: self._logger.error("Error in AlgebraicProtocol: %s", traceback.format_exc()) self.transport.close()
def _createConnectionEntry(self): identity = self.identityProducer.createIdentity() fieldId = self._currentTypeMap().fieldIdFor("core", "Connection", " exists") exists_key = ObjectFieldId(objId=identity, fieldId=fieldId) exists_index = IndexId(fieldId=fieldId, indexValue=indexValueFor(bool, True)) identityRoot = self.allocateNewIdentityRoot() self._handleNewTransaction(None, {exists_key: serialize(bool, True)}, {exists_index: set([identity])}, {}, [], [], self._cur_transaction_num) return core_schema.Connection.fromIdentity(identity), identityRoot
def test_tuples(self): # tuples are a compound with indices on item numbers T = TupleOf(int) self.assertEqual(serialize(T, T(())), EMPTY(0)) # A single element tuple is SINGLE + 0=VARINT + count + 0=VARINT + value self.assertEqual( serialize(T, T((1, ))), BEGIN_COMPOUND(0) + VARINT(0) + unsignedVarint(1) + VARINT(0) + signedVarint(1) + END_COMPOUND()) self.assertEqual( serialize(T, T((123, 124))), BEGIN_COMPOUND(0) + (VARINT(0) + unsignedVarint(2) + VARINT(0) + signedVarint(123) + VARINT(0) + signedVarint(124)) + END_COMPOUND()) self.assertEqual( serialize(T, T((123, 124, 125))), BEGIN_COMPOUND(0) + (VARINT(0) + unsignedVarint(3) + VARINT(0) + signedVarint(123) + VARINT(0) + signedVarint(124) + VARINT(0) + signedVarint(125)) + END_COMPOUND())
def test_serialize_stream_complex(self): T = OneOf(None, float, str, int, ListOf(int)) for items in [ (1, 2), ("hi", None, 10, [1, 2, 3, 4]), ()]: self.assertEqual( serializeStream(T, items), b"".join([serialize(T, x) for x in items]) ) self.assertEqual( deserializeStream(T, serializeStream(T, items)), TupleOf(T)(items) )
def test_conversion_of_binary_compatible(self): class T1(NamedTuple(a=int)): pass class T2(NamedTuple(a=int)): pass class T1Comp(NamedTuple(d=ConstDict(str, T1))): pass class T2Comp(NamedTuple(d=ConstDict(str, T1))): pass aT1C = T1Comp(d={'a': T1(a=10)}) self.assertEqual(T2Comp(aT1C).d['a'].a, 10) self.assertEqual(aT1C, deserialize(T1Comp, serialize(T2Comp, aT1C)))
def test_serialize_classes(self): class AClass(Class): x = Member(int) y = Member(float) T = Tuple(AClass, AClass) a = AClass(x=10, y=20.0) a2, a2_copy = deserialize(T, serialize(T, (a, a))) self.assertEqual(a2.x, 10) a2.x = 300 self.assertEqual(a2_copy.x, 300) a2_copy = None self.assertEqual(refcount(a2), 1)
def test_recursive_list(self): L = Forward("L") L = L.define(ListOf(OneOf(int, L))) listInst = L() listInst.append(10) listInst.append(listInst) self.assertEqual( serialize(L, listInst), BEGIN_COMPOUND(0) + ( VARINT(0) + unsignedVarint(0) + # the ID VARINT(0) + unsignedVarint(2) + # the size SINGLE(0) + VARINT(0) + signedVarint(10) + SINGLE(0) + SINGLE(1) + ( # a compound object encoding the second element of the one-of VARINT(0) + unsignedVarint(0) # the ID and nothing else )) + END_COMPOUND())
def test_serialize_class_instance(self): class A: def __init__(self, x): self.x = x def f(self): return b"an embedded string" ts = SerializationContext({'A': A}) serialization = ts.serialize(A(10)) self.assertTrue(b'an embedded string' not in serialization) anA = ts.deserialize(serialization) self.assertEqual(anA.x, 10) anA2 = deserialize(A, serialize(A, A(10), ts), ts) self.assertEqual(anA2.x, 10)