def test_Serialized(): s = Serialized(*serialize(123)) t = Serialized(*serialize((1, 2))) u = Serialized(*serialize(123)) assert s == u assert not (s != u) assert s != t assert not (s == t)
def check_deserialize_roundtrip(addr): """ Sanity check round-tripping with "deserialize" on and off. """ # Test with long bytestrings, large enough to be transferred # as a separate payload _uncompressible = os.urandom(1024**2) * 4 # end size: 4 MB msg = { "op": "update", "x": _uncompressible, "to_ser": [to_serialize(_uncompressible)], "ser": Serialized(*serialize(_uncompressible)), } for should_deserialize in (True, False): a, b = yield get_comm_pair(addr, deserialize=should_deserialize) yield a.write(msg) got = yield b.read() yield b.write(got) got = yield a.read() assert sorted(got) == sorted(msg) for k in ("op", "x"): assert got[k] == msg[k] if should_deserialize: assert isinstance(got["to_ser"][0], (bytes, bytearray)) assert isinstance(got["ser"], (bytes, bytearray)) else: assert isinstance(got["to_ser"][0], (to_serialize, Serialized)) assert isinstance(got["ser"], Serialized)
async def run_bench(protocol, nbytes, niters): data = np.random.randint(0, 255, size=nbytes, dtype=np.uint8) item = Serialized(*serialize(data)) if protocol == 'tcp': listener = listen('tcp://127.0.0.1', server_handle_comm, deserialize=False) else: listener = listen('ucx://' + ucp.get_address(), server_handle_comm, deserialize=False) listener.start() start = clock() for i in range(niters): comm = await connect(listener.contact_address, deserialize=False) await comm.write({'op': 'ping', 'item': item}) msg = await comm.read() assert msg['op'] == 'pong' assert isinstance(msg['item'], Serialized) await comm.close() print('.', end='', flush=True) print() end = clock() listener.stop() dt = end - start rate = len(data) * niters / dt print("duration: %s => rate: %d MB/s" % (dt, rate / 1e6))
def test_nested_deserialize(): x = {'op': 'update', 'x': [to_serialize(123), to_serialize(456), 789], 'y': {'a': ['abc', Serialized(*serialize('def'))], 'b': b'ghi'} } x_orig = copy.deepcopy(x) assert nested_deserialize(x) == {'op': 'update', 'x': [123, 456, 789], 'y': {'a': ['abc', 'def'], 'b': b'ghi'} } assert x == x_orig # x wasn't mutated
def test_nested_deserialize(): x = { "op": "update", "x": [to_serialize(123), to_serialize(456), 789], "y": {"a": ["abc", Serialized(*serialize("def"))], "b": b"ghi"}, } x_orig = copy.deepcopy(x) assert nested_deserialize(x) == { "op": "update", "x": [123, 456, 789], "y": {"a": ["abc", "def"], "b": b"ghi"}, } assert x == x_orig # x wasn't mutated
def test_serialize_iterate_collection(): # Use iterate_collection to ensure elements of # a collection will be serialized seperately arr = "special-data" sarr = Serialized(*serialize(arr)) sdarr = to_serialize(arr) task1 = (0, sarr, "('fake-key', 3)", None) task2 = (0, sdarr, "('fake-key', 3)", None) expect = (0, arr, "('fake-key', 3)", None) # Check serialize/deserialize directly assert deserialize(*serialize(task1, iterate_collection=True)) == expect assert deserialize(*serialize(task2, iterate_collection=True)) == expect
def check_deserialize(addr): """ Check the "deserialize" flag on connect() and listen(). """ # Test with Serialize and Serialized objects msg = { "op": "update", "x": b"abc", "to_ser": [to_serialize(123)], "ser": Serialized(*serialize(456)), } msg_orig = msg.copy() def check_out_false(out_value): # Check output with deserialize=False out_value = out_value.copy( ) # in case transport passed the object as-is to_ser = out_value.pop("to_ser") ser = out_value.pop("ser") expected_msg = msg_orig.copy() del expected_msg["ser"] del expected_msg["to_ser"] assert out_value == expected_msg assert isinstance(ser, Serialized) assert deserialize(ser.header, ser.frames) == 456 assert isinstance(to_ser, list) (to_ser, ) = to_ser # The to_serialize() value could have been actually serialized # or not (it's a transport-specific optimization) if isinstance(to_ser, Serialized): assert deserialize(to_ser.header, to_ser.frames) == 123 else: assert to_ser == to_serialize(123) def check_out_true(out_value): # Check output with deserialize=True expected_msg = msg.copy() expected_msg["ser"] = 456 expected_msg["to_ser"] = [123] assert out_value == expected_msg yield check_listener_deserialize(addr, False, msg, check_out_false) yield check_connector_deserialize(addr, False, msg, check_out_false) yield check_listener_deserialize(addr, True, msg, check_out_true) yield check_connector_deserialize(addr, True, msg, check_out_true) # Test with long bytestrings, large enough to be transferred # as a separate payload _uncompressible = os.urandom(1024**2) * 4 # end size: 8 MB msg = { "op": "update", "x": _uncompressible, "to_ser": [to_serialize(_uncompressible)], "ser": Serialized(*serialize(_uncompressible)), } msg_orig = msg.copy() def check_out(deserialize_flag, out_value): # Check output with deserialize=False assert sorted(out_value) == sorted(msg_orig) out_value = out_value.copy( ) # in case transport passed the object as-is to_ser = out_value.pop("to_ser") ser = out_value.pop("ser") expected_msg = msg_orig.copy() del expected_msg["ser"] del expected_msg["to_ser"] assert out_value == expected_msg if deserialize_flag: assert isinstance(ser, (bytes, bytearray)) assert bytes(ser) == _uncompressible else: assert isinstance(ser, Serialized) assert deserialize(ser.header, ser.frames) == _uncompressible assert isinstance(to_ser, list) (to_ser, ) = to_ser # The to_serialize() value could have been actually serialized # or not (it's a transport-specific optimization) if isinstance(to_ser, Serialized): assert deserialize(to_ser.header, to_ser.frames) == _uncompressible else: assert to_ser == to_serialize(_uncompressible) yield check_listener_deserialize(addr, False, msg, partial(check_out, False)) yield check_connector_deserialize(addr, False, msg, partial(check_out, False)) yield check_listener_deserialize(addr, True, msg, partial(check_out, True)) yield check_connector_deserialize(addr, True, msg, partial(check_out, True))
def check_deserialize(addr): # Test with Serialize and Serialized objects msg = {'op': 'update', 'x': b'abc', 'to_ser': [to_serialize(123)], 'ser': Serialized(*serialize(456)), } msg_orig = msg.copy() def check_out_false(out_value): # Check output with deserialize=False out_value = out_value.copy() # in case transport passed the object as-is to_ser = out_value.pop('to_ser') ser = out_value.pop('ser') expected_msg = msg_orig.copy() del expected_msg['ser'] del expected_msg['to_ser'] assert out_value == expected_msg assert isinstance(ser, Serialized) assert deserialize(ser.header, ser.frames) == 456 assert isinstance(to_ser, list) to_ser, = to_ser # The to_serialize() value could have been actually serialized # or not (it's a transport-specific optimization) if isinstance(to_ser, Serialized): assert deserialize(to_ser.header, to_ser.frames) == 123 else: assert to_ser == to_serialize(123) def check_out_true(out_value): # Check output with deserialize=True expected_msg = msg.copy() expected_msg['ser'] = 456 expected_msg['to_ser'] = [123] assert out_value == expected_msg yield check_listener_deserialize(addr, False, msg, check_out_false) yield check_connector_deserialize(addr, False, msg, check_out_false) yield check_listener_deserialize(addr, True, msg, check_out_true) yield check_connector_deserialize(addr, True, msg, check_out_true) # Test with a long bytestring msg = {'op': 'update', 'x': b'abc', 'y': b'def\n' * (2 ** 20), } msg_orig = msg.copy() def check_out(out_value): assert out_value == msg_orig yield check_listener_deserialize(addr, False, msg, check_out) yield check_connector_deserialize(addr, False, msg, check_out) yield check_listener_deserialize(addr, True, msg, check_out) yield check_connector_deserialize(addr, True, msg, check_out)
def to_serialized(obj): from distributed.protocol import Serialized, serialize return Serialized(*serialize(obj))