def test_Serialized():
s = Serialized(*serialize(123))
t = Serialized(*serialize((1, 2)))
u = Serialized(*serialize(123))
assert s == u
assert not (s != u)
assert s != t
assert not (s == t)
def check_deserialize_roundtrip(addr):
"""
Sanity check round-tripping with "deserialize" on and off.
"""
# Test with long bytestrings, large enough to be transferred
# as a separate payload
_uncompressible = os.urandom(1024**2) * 4 # end size: 4 MB
msg = {
"op": "update",
"x": _uncompressible,
"to_ser": [to_serialize(_uncompressible)],
"ser": Serialized(*serialize(_uncompressible)),
}
for should_deserialize in (True, False):
a, b = yield get_comm_pair(addr, deserialize=should_deserialize)
yield a.write(msg)
got = yield b.read()
yield b.write(got)
got = yield a.read()
assert sorted(got) == sorted(msg)
for k in ("op", "x"):
assert got[k] == msg[k]
if should_deserialize:
assert isinstance(got["to_ser"][0], (bytes, bytearray))
assert isinstance(got["ser"], (bytes, bytearray))
else:
assert isinstance(got["to_ser"][0], (to_serialize, Serialized))
assert isinstance(got["ser"], Serialized)
async def run_bench(protocol, nbytes, niters):
data = np.random.randint(0, 255, size=nbytes, dtype=np.uint8)
item = Serialized(*serialize(data))
if protocol == 'tcp':
listener = listen('tcp://127.0.0.1',
server_handle_comm,
deserialize=False)
else:
listener = listen('ucx://' + ucp.get_address(),
server_handle_comm,
deserialize=False)
listener.start()
start = clock()
for i in range(niters):
comm = await connect(listener.contact_address, deserialize=False)
await comm.write({'op': 'ping', 'item': item})
msg = await comm.read()
assert msg['op'] == 'pong'
assert isinstance(msg['item'], Serialized)
await comm.close()
print('.', end='', flush=True)
print()
end = clock()
listener.stop()
dt = end - start
rate = len(data) * niters / dt
print("duration: %s => rate: %d MB/s" % (dt, rate / 1e6))
def test_nested_deserialize():
x = {'op': 'update',
'x': [to_serialize(123), to_serialize(456), 789],
'y': {'a': ['abc', Serialized(*serialize('def'))],
'b': b'ghi'}
}
x_orig = copy.deepcopy(x)
assert nested_deserialize(x) == {'op': 'update',
'x': [123, 456, 789],
'y': {'a': ['abc', 'def'],
'b': b'ghi'}
}
assert x == x_orig # x wasn't mutated
def test_nested_deserialize():
x = {
"op": "update",
"x": [to_serialize(123), to_serialize(456), 789],
"y": {"a": ["abc", Serialized(*serialize("def"))], "b": b"ghi"},
}
x_orig = copy.deepcopy(x)
assert nested_deserialize(x) == {
"op": "update",
"x": [123, 456, 789],
"y": {"a": ["abc", "def"], "b": b"ghi"},
}
assert x == x_orig # x wasn't mutated
def test_serialize_iterate_collection():
# Use iterate_collection to ensure elements of
# a collection will be serialized seperately
arr = "special-data"
sarr = Serialized(*serialize(arr))
sdarr = to_serialize(arr)
task1 = (0, sarr, "('fake-key', 3)", None)
task2 = (0, sdarr, "('fake-key', 3)", None)
expect = (0, arr, "('fake-key', 3)", None)
# Check serialize/deserialize directly
assert deserialize(*serialize(task1, iterate_collection=True)) == expect
assert deserialize(*serialize(task2, iterate_collection=True)) == expect
def check_deserialize(addr):
"""
Check the "deserialize" flag on connect() and listen().
"""
# Test with Serialize and Serialized objects
msg = {
"op": "update",
"x": b"abc",
"to_ser": [to_serialize(123)],
"ser": Serialized(*serialize(456)),
}
msg_orig = msg.copy()
def check_out_false(out_value):
# Check output with deserialize=False
out_value = out_value.copy(
) # in case transport passed the object as-is
to_ser = out_value.pop("to_ser")
ser = out_value.pop("ser")
expected_msg = msg_orig.copy()
del expected_msg["ser"]
del expected_msg["to_ser"]
assert out_value == expected_msg
assert isinstance(ser, Serialized)
assert deserialize(ser.header, ser.frames) == 456
assert isinstance(to_ser, list)
(to_ser, ) = to_ser
# The to_serialize() value could have been actually serialized
# or not (it's a transport-specific optimization)
if isinstance(to_ser, Serialized):
assert deserialize(to_ser.header, to_ser.frames) == 123
else:
assert to_ser == to_serialize(123)
def check_out_true(out_value):
# Check output with deserialize=True
expected_msg = msg.copy()
expected_msg["ser"] = 456
expected_msg["to_ser"] = [123]
assert out_value == expected_msg
yield check_listener_deserialize(addr, False, msg, check_out_false)
yield check_connector_deserialize(addr, False, msg, check_out_false)
yield check_listener_deserialize(addr, True, msg, check_out_true)
yield check_connector_deserialize(addr, True, msg, check_out_true)
# Test with long bytestrings, large enough to be transferred
# as a separate payload
_uncompressible = os.urandom(1024**2) * 4 # end size: 8 MB
msg = {
"op": "update",
"x": _uncompressible,
"to_ser": [to_serialize(_uncompressible)],
"ser": Serialized(*serialize(_uncompressible)),
}
msg_orig = msg.copy()
def check_out(deserialize_flag, out_value):
# Check output with deserialize=False
assert sorted(out_value) == sorted(msg_orig)
out_value = out_value.copy(
) # in case transport passed the object as-is
to_ser = out_value.pop("to_ser")
ser = out_value.pop("ser")
expected_msg = msg_orig.copy()
del expected_msg["ser"]
del expected_msg["to_ser"]
assert out_value == expected_msg
if deserialize_flag:
assert isinstance(ser, (bytes, bytearray))
assert bytes(ser) == _uncompressible
else:
assert isinstance(ser, Serialized)
assert deserialize(ser.header, ser.frames) == _uncompressible
assert isinstance(to_ser, list)
(to_ser, ) = to_ser
# The to_serialize() value could have been actually serialized
# or not (it's a transport-specific optimization)
if isinstance(to_ser, Serialized):
assert deserialize(to_ser.header,
to_ser.frames) == _uncompressible
else:
assert to_ser == to_serialize(_uncompressible)
yield check_listener_deserialize(addr, False, msg,
partial(check_out, False))
yield check_connector_deserialize(addr, False, msg,
partial(check_out, False))
yield check_listener_deserialize(addr, True, msg, partial(check_out, True))
yield check_connector_deserialize(addr, True, msg,
partial(check_out, True))
def check_deserialize(addr):
# Test with Serialize and Serialized objects
msg = {'op': 'update',
'x': b'abc',
'to_ser': [to_serialize(123)],
'ser': Serialized(*serialize(456)),
}
msg_orig = msg.copy()
def check_out_false(out_value):
# Check output with deserialize=False
out_value = out_value.copy() # in case transport passed the object as-is
to_ser = out_value.pop('to_ser')
ser = out_value.pop('ser')
expected_msg = msg_orig.copy()
del expected_msg['ser']
del expected_msg['to_ser']
assert out_value == expected_msg
assert isinstance(ser, Serialized)
assert deserialize(ser.header, ser.frames) == 456
assert isinstance(to_ser, list)
to_ser, = to_ser
# The to_serialize() value could have been actually serialized
# or not (it's a transport-specific optimization)
if isinstance(to_ser, Serialized):
assert deserialize(to_ser.header, to_ser.frames) == 123
else:
assert to_ser == to_serialize(123)
def check_out_true(out_value):
# Check output with deserialize=True
expected_msg = msg.copy()
expected_msg['ser'] = 456
expected_msg['to_ser'] = [123]
assert out_value == expected_msg
yield check_listener_deserialize(addr, False, msg, check_out_false)
yield check_connector_deserialize(addr, False, msg, check_out_false)
yield check_listener_deserialize(addr, True, msg, check_out_true)
yield check_connector_deserialize(addr, True, msg, check_out_true)
# Test with a long bytestring
msg = {'op': 'update',
'x': b'abc',
'y': b'def\n' * (2 ** 20),
}
msg_orig = msg.copy()
def check_out(out_value):
assert out_value == msg_orig
yield check_listener_deserialize(addr, False, msg, check_out)
yield check_connector_deserialize(addr, False, msg, check_out)
yield check_listener_deserialize(addr, True, msg, check_out)
yield check_connector_deserialize(addr, True, msg, check_out)
def to_serialized(obj):
from distributed.protocol import Serialized, serialize
return Serialized(*serialize(obj))