async def recv(eps, in_nparts: Dict[int, int],
out_parts_list: List[List[List[DataFrame]]]):
"""Notice, received items are appended to `out_parts_list`"""
futures = []
for rank, ep in eps.items():
if rank in in_nparts:
futures.append(ep.read())
out_parts_list.extend(nested_deserialize(await asyncio.gather(*futures)))
async def recv(eps, in_nparts: Dict[int, int],
out_parts_list: List[List[List[DataFrame]]]):
"""Notice, received items are appended to `out_parts_list`"""
futures = []
for rank, ep in eps.items():
if rank in in_nparts:
futures.append(ep.read())
# Notice, since Dask may convert lists to tuples, we convert them back into lists
out_parts_list.extend(
[[y for y in x]
for x in nested_deserialize(await asyncio.gather(*futures))])
async def read(self, deserializers="ignored"):
if self._closed:
raise CommClosedError()
msg = await self._read_q.get()
if msg is _EOF:
self._closed = True
self._finalizer.detach()
raise CommClosedError()
if self.deserialize:
msg = nested_deserialize(msg)
return msg
def test_nested_deserialize():
x = {'op': 'update',
'x': [to_serialize(123), to_serialize(456), 789],
'y': {'a': ['abc', Serialized(*serialize('def'))],
'b': b'ghi'}
}
x_orig = copy.deepcopy(x)
assert nested_deserialize(x) == {'op': 'update',
'x': [123, 456, 789],
'y': {'a': ['abc', 'def'],
'b': b'ghi'}
}
assert x == x_orig # x wasn't mutated
def test_nested_deserialize():
x = {
"op": "update",
"x": [to_serialize(123), to_serialize(456), 789],
"y": {"a": ["abc", Serialized(*serialize("def"))], "b": b"ghi"},
}
x_orig = copy.deepcopy(x)
assert nested_deserialize(x) == {
"op": "update",
"x": [123, 456, 789],
"y": {"a": ["abc", "def"], "b": b"ghi"},
}
assert x == x_orig # x wasn't mutated
def test_nested_deserialize():
x = {'op': 'update',
'x': [to_serialize(123), to_serialize(456), 789],
'y': {'a': ['abc', Serialized(*serialize('def'))],
'b': b'ghi'}
}
x_orig = copy.deepcopy(x)
assert nested_deserialize(x) == {'op': 'update',
'x': [123, 456, 789],
'y': {'a': ['abc', 'def'],
'b': b'ghi'}
}
assert x == x_orig # x wasn't mutated
