def test_context_specific_serialization_class(c, s, a, b):
register_serialization(MyObject, my_dumps, my_loads)
try:
# Create the object on A, force communication to B
x = c.submit(MyObject, x=1, y=2, workers=a.address)
y = c.submit(lambda x: x, x, workers=b.address)
yield wait(y)
key = y.key
def check(dask_worker):
# Get the context from the object stored on B
my_obj = dask_worker.data[key]
return my_obj.context
result = yield c.run(check, workers=[b.address])
expected = {'sender': a.address, 'recipient': b.address}
assert result[b.address]['sender'] == a.address # see origin worker
z = yield y # bring object to local process
assert z.x == 1 and z.y == 2
assert z.context['sender'] == b.address
finally:
from distributed.protocol.serialize import class_serializers, typename
del class_serializers[typename(MyObject)]
async def test_context_specific_serialization_class(c, s, a, b):
register_serialization(MyObject, my_dumps, my_loads)
# Create the object on A, force communication to B
x = c.submit(MyObject, x=1, y=2, workers=a.address)
y = c.submit(lambda x: x, x, workers=b.address)
await wait(y)
key = y.key
def check(dask_worker):
# Get the context from the object stored on B
my_obj = dask_worker.data[key]
return my_obj.context
result = await c.run(check, workers=[b.address])
assert result[b.address]["sender"]["address"] == a.address # see origin worker
z = await y # bring object to local process
assert z.x == 1 and z.y == 2
assert z.context["sender"]["address"] == b.address
def test_context_specific_serialization_class(c, s, a, b):
register_serialization(MyObject, my_dumps, my_loads)
# Create the object on A, force communication to B
x = c.submit(MyObject, x=1, y=2, workers=a.address)
y = c.submit(lambda x: x, x, workers=b.address)
yield wait(y)
key = y.key
def check(dask_worker):
# Get the context from the object stored on B
my_obj = dask_worker.data[key]
return my_obj.context
result = yield c.run(check, workers=[b.address])
expected = {'sender': a.address, 'recipient': b.address}
assert result[b.address]['sender'] == a.address # see origin worker
z = yield y # bring object to local process
assert z.x == 1 and z.y == 2
assert z.context['sender'] == b.address
def __init__(self, data):
self.data = data
def __getstate__(self):
raise Exception("Not picklable")
def serialize_myobj(x):
return {}, [pickle.dumps(x.data)]
def deserialize_myobj(header, frames):
return MyObj(pickle.loads(frames[0]))
register_serialization(MyObj, serialize_myobj, deserialize_myobj)
def test_dumps_serialize():
for x in [123, [1, 2, 3, 4, 5, 6]]:
header, frames = serialize(x)
assert header["serializer"] == "pickle"
assert len(frames) == 1
result = deserialize(header, frames)
assert result == x
x = MyObj(123)
header, frames = serialize(x)
assert header["type"]
assert len(frames) == 1
def register_distributed_serializer(cls):
"""Register serialization methods for dask.distributed.
"""
_dp.register_serialization(cls, _serialize, _deserialize)
class MyObj(object):
def __init__(self, data):
self.data = data
def __getstate__(self):
raise Exception('Not picklable')
def serialize_myobj(x):
return {}, [pickle.dumps(x.data)]
def deserialize_myobj(header, frames):
return MyObj(pickle.loads(frames[0]))
register_serialization(MyObj, serialize_myobj, deserialize_myobj)
def test_dumps_serialize():
for x in [123, [1, 2, 3]]:
header, frames = serialize(x)
assert not header
assert len(frames) == 1
result = deserialize(header, frames)
assert result == x
x = MyObj(123)
header, frames = serialize(x)
assert header['type']
assert len(frames) == 1
def register_distributed_serializer(cls):
_dp.register_serialization(cls, _serialize, _deserialize)
