def df_gen():
# build up a dask graph to run all of the `convert` calls concurrently
# use a list to hold the requested key names to ensure that we return
# the results in the correct order
p = []
dsk = {}
for n, csv_ in enumerate(csvs):
key = 'p%d' % n
dsk[key] = f, csv_
p.append(key)
return concat(dsk_get(dsk, p))
def df_gen():
# build up a dask graph to run all of the `convert` calls concurrently
# use a list to hold the requested key names to ensure that we return
# the results in the correct order
p = []
dsk = {}
for n, csv_ in enumerate(csvs):
key = 'p%d' % n
dsk[key] = f, csv_
p.append(key)
return concat(dsk_get(dsk, p))
def __iter__(self):
if callable(self.data):
return self.data()
elif isinstance(self.data, list) and len(self.data) and callable(self.data[0]):
# If this is a set of callables, evaluate
# them using dask before returning an iterator for them
p = []
dsk = {}
for i, f in enumerate(self.data):
dsk["p%d" % i] = (f,)
p.append("p%d" % i)
self.data = dsk_get(dsk, p)
return iter(self.data)
def __iter__(self):
if callable(self.data):
return self.data()
elif (isinstance(self.data, list) and len(self.data)
and callable(self.data[0])):
# If this is a set of callables, evaluate
# them using dask before returning an iterator for them
p = []
dsk = {}
for i, f in enumerate(self.data):
dsk['p%d' % i] = (f, )
p.append('p%d' % i)
self.data = dsk_get(dsk, p)
return iter(self.data)
