def df_gen(): # build up a dask graph to run all of the `convert` calls concurrently # use a list to hold the requested key names to ensure that we return # the results in the correct order p = [] dsk = {} for n, csv_ in enumerate(csvs): key = 'p%d' % n dsk[key] = f, csv_ p.append(key) return concat(dsk_get(dsk, p))
def __iter__(self): if callable(self.data): return self.data() elif isinstance(self.data, list) and len(self.data) and callable(self.data[0]): # If this is a set of callables, evaluate # them using dask before returning an iterator for them p = [] dsk = {} for i, f in enumerate(self.data): dsk["p%d" % i] = (f,) p.append("p%d" % i) self.data = dsk_get(dsk, p) return iter(self.data)
def __iter__(self): if callable(self.data): return self.data() elif (isinstance(self.data, list) and len(self.data) and callable(self.data[0])): # If this is a set of callables, evaluate # them using dask before returning an iterator for them p = [] dsk = {} for i, f in enumerate(self.data): dsk['p%d' % i] = (f, ) p.append('p%d' % i) self.data = dsk_get(dsk, p) return iter(self.data)