async def recv(eps, in_nparts: Dict[int, int], out_parts_list: List[List[List[DataFrame]]]): """Notice, received items are appended to `out_parts_list`""" futures = [] for rank, ep in eps.items(): if rank in in_nparts: futures.append(ep.read()) out_parts_list.extend(nested_deserialize(await asyncio.gather(*futures)))
async def recv(eps, in_nparts: Dict[int, int], out_parts_list: List[List[List[DataFrame]]]): """Notice, received items are appended to `out_parts_list`""" futures = [] for rank, ep in eps.items(): if rank in in_nparts: futures.append(ep.read()) # Notice, since Dask may convert lists to tuples, we convert them back into lists out_parts_list.extend( [[y for y in x] for x in nested_deserialize(await asyncio.gather(*futures))])
async def read(self, deserializers="ignored"): if self._closed: raise CommClosedError() msg = await self._read_q.get() if msg is _EOF: self._closed = True self._finalizer.detach() raise CommClosedError() if self.deserialize: msg = nested_deserialize(msg) return msg
def test_nested_deserialize(): x = {'op': 'update', 'x': [to_serialize(123), to_serialize(456), 789], 'y': {'a': ['abc', Serialized(*serialize('def'))], 'b': b'ghi'} } x_orig = copy.deepcopy(x) assert nested_deserialize(x) == {'op': 'update', 'x': [123, 456, 789], 'y': {'a': ['abc', 'def'], 'b': b'ghi'} } assert x == x_orig # x wasn't mutated
def test_nested_deserialize(): x = { "op": "update", "x": [to_serialize(123), to_serialize(456), 789], "y": {"a": ["abc", Serialized(*serialize("def"))], "b": b"ghi"}, } x_orig = copy.deepcopy(x) assert nested_deserialize(x) == { "op": "update", "x": [123, 456, 789], "y": {"a": ["abc", "def"], "b": b"ghi"}, } assert x == x_orig # x wasn't mutated
def test_nested_deserialize(): x = {'op': 'update', 'x': [to_serialize(123), to_serialize(456), 789], 'y': {'a': ['abc', Serialized(*serialize('def'))], 'b': b'ghi'} } x_orig = copy.deepcopy(x) assert nested_deserialize(x) == {'op': 'update', 'x': [123, 456, 789], 'y': {'a': ['abc', 'def'], 'b': b'ghi'} } assert x == x_orig # x wasn't mutated