def test_itemgetter():
data = [1, 2, 3]
g = itemgetter(1)
assert g(data) == 2
g2 = pickle.loads(pickle.dumps(g))
assert g2(data) == 2
assert g2.index == 1
def test_itemgetter():
data = [1, 2, 3]
g = itemgetter(1)
assert g(data) == 2
g2 = pickle.loads(pickle.dumps(g))
assert g2(data) == 2
assert g2.index == 1
async def maybe_to_futures(args):
out = []
for arg in args:
arg_id = id(arg)
if arg_id in itemgetters:
out.append(itemgetters[arg_id])
continue
f = self.data_futures.get(arg_id, None)
if f is None and call_data_futures is not None:
try:
f = call_data_futures[arg]
except KeyError:
if is_weakrefable(arg) and sizeof(arg) > 1e3:
# Automatically scatter large objects to some of
# the workers to avoid duplicated data transfers.
# Rely on automated inter-worker data stealing if
# more workers need to reuse this data
# concurrently.
[f] = await self.client.scatter([arg],
asynchronous=True)
call_data_futures[arg] = f
if f is not None:
getter = itemgetter(len(collected_futures))
collected_futures.append(f)
itemgetters[arg_id] = getter
arg = getter
out.append(arg)
return out
