def maybe_to_futures(args): for arg in args: arg_id = id(arg) if arg_id in itemgetters: yield itemgetters[arg_id] continue f = self.data_futures.get(arg_id, None) if f is None and call_data_futures is not None: try: f = call_data_futures[arg] except KeyError: if is_weakrefable(arg) and sizeof(arg) > 1e3: # Automatically scatter large objects to some of # the workers to avoid duplicated data transfers. # Rely on automated inter-worker data stealing if # more workers need to reuse this data # concurrently. [f] = self.client.scatter([arg]) call_data_futures[arg] = f if f is not None: getter = itemgetter(len(collected_futures)) collected_futures.append(f) itemgetters[arg_id] = getter arg = getter yield arg
def maybe_to_futures(args): for arg in args: arg_id = id(arg) if arg_id in itemgetters: yield itemgetters[arg_id] continue f = self.data_futures.get(arg_id, None) if f is None and call_data_futures is not None: try: f = call_data_futures[arg] except KeyError: if is_weakrefable(arg) and sizeof(arg) > 1e3: # Automatically scatter large objects to some of # the workers to avoid duplicated data transfers. # Rely on automated inter-worker data stealing if # more workers need to reuse this data # concurrently. [f] = self.client.scatter([arg]) call_data_futures[arg] = f if f is not None: getter = itemgetter(len(collected_futures)) collected_futures.append(f) itemgetters[arg_id] = getter arg = getter yield arg