def test_context_specific_serialization_class(c, s, a, b): register_serialization(MyObject, my_dumps, my_loads) try: # Create the object on A, force communication to B x = c.submit(MyObject, x=1, y=2, workers=a.address) y = c.submit(lambda x: x, x, workers=b.address) yield wait(y) key = y.key def check(dask_worker): # Get the context from the object stored on B my_obj = dask_worker.data[key] return my_obj.context result = yield c.run(check, workers=[b.address]) expected = {'sender': a.address, 'recipient': b.address} assert result[b.address]['sender'] == a.address # see origin worker z = yield y # bring object to local process assert z.x == 1 and z.y == 2 assert z.context['sender'] == b.address finally: from distributed.protocol.serialize import class_serializers, typename del class_serializers[typename(MyObject)]
async def test_context_specific_serialization_class(c, s, a, b): register_serialization(MyObject, my_dumps, my_loads) # Create the object on A, force communication to B x = c.submit(MyObject, x=1, y=2, workers=a.address) y = c.submit(lambda x: x, x, workers=b.address) await wait(y) key = y.key def check(dask_worker): # Get the context from the object stored on B my_obj = dask_worker.data[key] return my_obj.context result = await c.run(check, workers=[b.address]) assert result[b.address]["sender"]["address"] == a.address # see origin worker z = await y # bring object to local process assert z.x == 1 and z.y == 2 assert z.context["sender"]["address"] == b.address
def test_context_specific_serialization_class(c, s, a, b): register_serialization(MyObject, my_dumps, my_loads) # Create the object on A, force communication to B x = c.submit(MyObject, x=1, y=2, workers=a.address) y = c.submit(lambda x: x, x, workers=b.address) yield wait(y) key = y.key def check(dask_worker): # Get the context from the object stored on B my_obj = dask_worker.data[key] return my_obj.context result = yield c.run(check, workers=[b.address]) expected = {'sender': a.address, 'recipient': b.address} assert result[b.address]['sender'] == a.address # see origin worker z = yield y # bring object to local process assert z.x == 1 and z.y == 2 assert z.context['sender'] == b.address
def __init__(self, data): self.data = data def __getstate__(self): raise Exception("Not picklable") def serialize_myobj(x): return {}, [pickle.dumps(x.data)] def deserialize_myobj(header, frames): return MyObj(pickle.loads(frames[0])) register_serialization(MyObj, serialize_myobj, deserialize_myobj) def test_dumps_serialize(): for x in [123, [1, 2, 3, 4, 5, 6]]: header, frames = serialize(x) assert header["serializer"] == "pickle" assert len(frames) == 1 result = deserialize(header, frames) assert result == x x = MyObj(123) header, frames = serialize(x) assert header["type"] assert len(frames) == 1
def register_distributed_serializer(cls): """Register serialization methods for dask.distributed. """ _dp.register_serialization(cls, _serialize, _deserialize)
class MyObj(object): def __init__(self, data): self.data = data def __getstate__(self): raise Exception('Not picklable') def serialize_myobj(x): return {}, [pickle.dumps(x.data)] def deserialize_myobj(header, frames): return MyObj(pickle.loads(frames[0])) register_serialization(MyObj, serialize_myobj, deserialize_myobj) def test_dumps_serialize(): for x in [123, [1, 2, 3]]: header, frames = serialize(x) assert not header assert len(frames) == 1 result = deserialize(header, frames) assert result == x x = MyObj(123) header, frames = serialize(x) assert header['type'] assert len(frames) == 1
def register_distributed_serializer(cls): _dp.register_serialization(cls, _serialize, _deserialize)